token-vocabs 0.3.0 → 0.5.0

package/README.md

@@ -23,7 +23,7 @@ Count tokens or inspect token IDs across several modern tokenizer families from
 - exact golden outputs for the core sample fixture
 - one Brotli-compressed MessagePack asset bundle per model
 - browser Brotli decompression with a bundled JS fallback where native stream support is missing
-- Rolldown browser builds that emit binary `.msgpack.br` vocabulary bundles, shared chunks and the required WASM asset
+- Rolldown browser builds that emit binary vocabulary bundles, shared chunks and the required WASM asset
 - async auto-loading API plus loaded-only sync helpers
 - one small single-model API for counts, token IDs and byte offsets
 - generated tokenizer assets via `bun run fetch`
@@ -138,7 +138,7 @@ Exports model metadata, including the original upstream source URLs used by `bun
 
 Browser entry with the same `count()`, `countLoaded()`, `tokenize()`, `tokenizeLoaded()`, `load()` and `free()` API as the desktop entry.
 
-It loads the `.msgpack.br` asset bundles via `fetch()`.
+It loads the `.bin` asset bundles via `fetch()`.
 
 ### `token-vocabs/browser/all`
 
@@ -165,6 +165,6 @@ console.dir(countLoaded('mind goblin', 'deepseek'))
 ## Notes
 
 - `sdxl` intentionally implements the shared CLIP BPE core used by SDXL without auto-adding BOS/EOS tokens.
-- GPT uses `tiktoken`’s built-in `o200k_base` implementation, but the upstream encoder payload is still fetched and converted to MessagePack for completeness.
-- Structured tokenizer payloads are stored inside per-model `.msgpack.br` bundles and decompressed after loading.
+- GPT uses `tiktoken` lite plus a vendored `o200k_base` model string, so the browser WASM stays lean and the vocabulary still lives in the regular per-model asset bundle.
+- Structured tokenizer payloads are stored inside per-model `.bin` bundles and decompressed after loading.
 - Tokenizer assets are large. That is inherent to exact offline tokenization.

package/chunks/main.js

@@ -1,2 +1,2 @@
-import{n as e,r as t}from"./chunk.js";var n;try{n=new TextDecoder}catch{}var r,i,a=0;const o=[];var s=o,c=0,l={},u,d,f=0,p=0,m,h,g=[],_,ee={useRecords:!1,mapsAsObjects:!0};const te=new class{};te.name=`MessagePack 0xC1`;var v=!1,ne=2,y=class e{constructor(e){e&&(e.useRecords===!1&&e.mapsAsObjects===void 0&&(e.mapsAsObjects=!0),e.sequential&&e.trusted!==!1&&(e.trusted=!0,!e.structures&&e.useRecords!=0&&(e.structures=[],e.maxSharedStructures||=0)),e.structures?e.structures.sharedLength=e.structures.length:e.getStructures&&((e.structures=[]).uninitialized=!0,e.structures.sharedLength=0),e.int64AsNumber&&(e.int64AsType=`number`)),Object.assign(this,e)}unpack(t,n){if(r)return Ee(()=>(De(),this?this.unpack(t,n):e.prototype.unpack.call(ee,t,n)));!t.buffer&&t.constructor===ArrayBuffer&&(t=typeof Buffer<`u`?Buffer.from(t):new Uint8Array(t)),typeof n==`object`?(i=n.end||t.length,a=n.start||0):(a=0,i=n>-1?n:t.length),c=0,p=0,d=null,s=o,m=null,r=t;try{_=t.dataView||=new DataView(t.buffer,t.byteOffset,t.byteLength)}catch(e){throw r=null,t instanceof Uint8Array?e:Error(`Source must be a Uint8Array or Buffer but was a `+(t&&typeof t==`object`?t.constructor.name:typeof t))}if(this instanceof e){if(l=this,this.structures)return u=this.structures,b(n);(!u||u.length>0)&&(u=[])}else l=ee,(!u||u.length>0)&&(u=[]);return b(n)}unpackMultiple(e,t){let n,r=0;try{v=!0;let i=e.length,o=this?this.unpack(e,i):T.unpack(e,i);if(t){if(t(o,r,a)===!1)return;for(;a<i;)if(r=a,t(b(),r,a)===!1)return}else{for(n=[o];a<i;)r=a,n.push(b());return n}}catch(e){throw e.lastPosition=r,e.values=n,e}finally{v=!1,De()}}_mergeStructures(e,t){this._onLoadedStructures&&(e=this._onLoadedStructures(e)),e||=[],Object.isFrozen(e)&&(e=e.map(e=>e.slice(0)));for(let t=0,n=e.length;t<n;t++){let n=e[t];n&&(n.isShared=!0,t>=32&&(n.highByte=t-32>>5))}e.sharedLength=e.length;for(let n in t||[])if(n>=0){let r=e[n],i=t[n];i&&(r&&((e.restoreStructures||=[])[n]=r),e[n]=i)}return this.structures=e}decode(e,t){return this.unpack(e,t)}};function b(e){try{if(!l.trusted&&!v){let e=u.sharedLength||0;e<u.length&&(u.length=e)}let t;if(l._readStruct&&r[a]<64&&r[a]>=32?(t=l._readStruct(r,a,i),r=null,!(e&&e.lazy)&&t&&(t=t.toJSON()),a=i):t=x(),m&&=(a=m.postBundlePosition,null),v&&(u.restoreStructures=null),a==i)u&&u.restoreStructures&&re(),u=null,r=null,h&&=null;else if(a>i)throw Error(`Unexpected end of MessagePack data`);else if(!v){let e;try{e=JSON.stringify(t,(e,t)=>typeof t==`bigint`?`${t}n`:t).slice(0,100)}catch(t){e=`(JSON view not available `+t+`)`}throw Error(`Data read, but end of buffer not reached `+e)}return t}catch(e){throw u&&u.restoreStructures&&re(),De(),(e instanceof RangeError||e.message.startsWith(`Unexpected end of buffer`)||a>i)&&(e.incomplete=!0),e}}function re(){for(let e in u.restoreStructures)u[e]=u.restoreStructures[e];u.restoreStructures=null}function x(){let e=r[a++];if(e<160)if(e<128){if(e<64)return e;{let t=u[e&63]||l.getStructures&&se()[e&63];return t?(t.read||=ae(t,e&63),t.read()):e}}else if(e<144)if(e-=128,l.mapsAsObjects){let t={};for(let n=0;n<e;n++){let e=ye();e===`__proto__`&&(e=`__proto_`),t[e]=x()}return t}else{let t=new Map;for(let n=0;n<e;n++)t.set(x(),x());return t}else{e-=144;let t=Array(e);for(let n=0;n<e;n++)t[n]=x();return l.freezeData?Object.freeze(t):t}else if(e<192){let t=e-160;if(p>=a)return d.slice(a-f,(a+=t)-f);if(p==0&&i<140){let e=t<16?he(t):me(t);if(e!=null)return e}return ce(t)}else{let t;switch(e){case 192:return null;case 193:return m?(t=x(),t>0?m[1].slice(m.position1,m.position1+=t):m[0].slice(m.position0,m.position0-=t)):te;case 194:return!1;case 195:return!0;case 196:if(t=r[a++],t===void 0)throw Error(`Unexpected end of buffer`);return _e(t);case 197:return t=_.getUint16(a),a+=2,_e(t);case 198:return t=_.getUint32(a),a+=4,_e(t);case 199:return w(r[a++]);case 200:return t=_.getUint16(a),a+=2,w(t);case 201:return t=_.getUint32(a),a+=4,w(t);case 202:if(t=_.getFloat32(a),l.useFloat32>2){let e=Oe[(r[a]&127)<<1|r[a+1]>>7];return a+=4,(e*t+(t>0?.5:-.5)>>0)/e}return a+=4,t;case 203:return t=_.getFloat64(a),a+=8,t;case 204:return r[a++];case 205:return t=_.getUint16(a),a+=2,t;case 206:return t=_.getUint32(a),a+=4,t;case 207:return l.int64AsType===`number`?(t=_.getUint32(a)*4294967296,t+=_.getUint32(a+4)):l.int64AsType===`string`?t=_.getBigUint64(a).toString():l.int64AsType===`auto`?(t=_.getBigUint64(a),t<=BigInt(2)<<BigInt(52)&&(t=Number(t))):t=_.getBigUint64(a),a+=8,t;case 208:return _.getInt8(a++);case 209:return t=_.getInt16(a),a+=2,t;case 210:return t=_.getInt32(a),a+=4,t;case 211:return l.int64AsType===`number`?(t=_.getInt32(a)*4294967296,t+=_.getUint32(a+4)):l.int64AsType===`string`?t=_.getBigInt64(a).toString():l.int64AsType===`auto`?(t=_.getBigInt64(a),t>=BigInt(-2)<<BigInt(52)&&t<=BigInt(2)<<BigInt(52)&&(t=Number(t))):t=_.getBigInt64(a),a+=8,t;case 212:if(t=r[a++],t==114)return xe(r[a++]&63);{let e=g[t];if(e)return e.read?(a++,e.read(x())):e.noBuffer?(a++,e()):e(r.subarray(a,++a));throw Error(`Unknown extension `+t)}case 213:return t=r[a],t==114?(a++,xe(r[a++]&63,r[a++])):w(2);case 214:return w(4);case 215:return w(8);case 216:return w(16);case 217:return t=r[a++],p>=a?d.slice(a-f,(a+=t)-f):le(t);case 218:return t=_.getUint16(a),a+=2,p>=a?d.slice(a-f,(a+=t)-f):ue(t);case 219:return t=_.getUint32(a),a+=4,p>=a?d.slice(a-f,(a+=t)-f):de(t);case 220:return t=_.getUint16(a),a+=2,fe(t);case 221:return t=_.getUint32(a),a+=4,fe(t);case 222:return t=_.getUint16(a),a+=2,pe(t);case 223:return t=_.getUint32(a),a+=4,pe(t);default:if(e>=224)return e-256;if(e===void 0){let e=Error(`Unexpected end of MessagePack data`);throw e.incomplete=!0,e}throw Error(`Unknown MessagePack token `+e)}}}const ie=/^[a-zA-Z_$][a-zA-Z\d_$]*$/;function ae(e,t){function n(){if(n.count++>ne){let r;try{r=e.read=Function(`r`,`return function(){return `+(l.freezeData?`Object.freeze`:``)+`({`+e.map(e=>e===`__proto__`?`__proto_:r()`:ie.test(e)?e+`:r()`:`[`+JSON.stringify(e)+`]:r()`).join(`,`)+`})}`)(x)}catch{return ne=1/0,n()}return e.highByte===0&&(e.read=oe(t,e.read)),r()}let r={};for(let t=0,n=e.length;t<n;t++){let n=e[t];n===`__proto__`&&(n=`__proto_`),r[n]=x()}return l.freezeData?Object.freeze(r):r}return n.count=0,e.highByte===0?oe(t,n):n}const oe=(e,t)=>function(){let n=r[a++];if(n===0)return t();let i=e<32?-(e+(n<<5)):e+(n<<5),o=u[i]||se()[i];if(!o)throw Error(`Record id is not defined for `+i);return o.read||=ae(o,e),o.read()};function se(){let e=Ee(()=>(r=null,l.getStructures()));return u=l._mergeStructures(e,u)}var ce=S,le=S,ue=S,de=S;function S(e){let t;if(e<16&&(t=he(e)))return t;if(e>64&&n)return n.decode(r.subarray(a,a+=e));let i=a+e,o=[];for(t=``;a<i;){let e=r[a++];if(!(e&128))o.push(e);else if((e&224)==192){let t=r[a++]&63,n=(e&31)<<6|t;n<128?o.push(65533):o.push(n)}else if((e&240)==224){let t=r[a++]&63,n=r[a++]&63,i=(e&31)<<12|t<<6|n;i<2048||i>=55296&&i<=57343?o.push(65533):o.push(i)}else if((e&248)==240){let t=r[a++]&63,n=r[a++]&63,i=r[a++]&63,s=(e&7)<<18|t<<12|n<<6|i;s<65536||s>1114111?o.push(65533):s>65535?(s-=65536,o.push(s>>>10&1023|55296),s=56320|s&1023,o.push(s)):o.push(s)}else o.push(65533);o.length>=4096&&(t+=C.apply(String,o),o.length=0)}return o.length>0&&(t+=C.apply(String,o)),t}function fe(e){let t=Array(e);for(let n=0;n<e;n++)t[n]=x();return l.freezeData?Object.freeze(t):t}function pe(e){if(l.mapsAsObjects){let t={};for(let n=0;n<e;n++){let e=ye();e===`__proto__`&&(e=`__proto_`),t[e]=x()}return t}else{let t=new Map;for(let n=0;n<e;n++)t.set(x(),x());return t}}var C=String.fromCharCode;function me(e){let t=a,n=Array(e);for(let i=0;i<e;i++){let e=r[a++];if((e&128)>0){a=t;return}n[i]=e}return C.apply(String,n)}function he(e){if(e<4)if(e<2){if(e===0)return``;{let e=r[a++];if((e&128)>1){--a;return}return C(e)}}else{let t=r[a++],n=r[a++];if((t&128)>0||(n&128)>0){a-=2;return}if(e<3)return C(t,n);let i=r[a++];if((i&128)>0){a-=3;return}return C(t,n,i)}else{let t=r[a++],n=r[a++],i=r[a++],o=r[a++];if((t&128)>0||(n&128)>0||(i&128)>0||(o&128)>0){a-=4;return}if(e<6){if(e===4)return C(t,n,i,o);{let e=r[a++];if((e&128)>0){a-=5;return}return C(t,n,i,o,e)}}else if(e<8){let s=r[a++],c=r[a++];if((s&128)>0||(c&128)>0){a-=6;return}if(e<7)return C(t,n,i,o,s,c);let l=r[a++];if((l&128)>0){a-=7;return}return C(t,n,i,o,s,c,l)}else{let s=r[a++],c=r[a++],l=r[a++],u=r[a++];if((s&128)>0||(c&128)>0||(l&128)>0||(u&128)>0){a-=8;return}if(e<10){if(e===8)return C(t,n,i,o,s,c,l,u);{let e=r[a++];if((e&128)>0){a-=9;return}return C(t,n,i,o,s,c,l,u,e)}}else if(e<12){let d=r[a++],f=r[a++];if((d&128)>0||(f&128)>0){a-=10;return}if(e<11)return C(t,n,i,o,s,c,l,u,d,f);let p=r[a++];if((p&128)>0){a-=11;return}return C(t,n,i,o,s,c,l,u,d,f,p)}else{let d=r[a++],f=r[a++],p=r[a++],m=r[a++];if((d&128)>0||(f&128)>0||(p&128)>0||(m&128)>0){a-=12;return}if(e<14){if(e===12)return C(t,n,i,o,s,c,l,u,d,f,p,m);{let e=r[a++];if((e&128)>0){a-=13;return}return C(t,n,i,o,s,c,l,u,d,f,p,m,e)}}else{let h=r[a++],g=r[a++];if((h&128)>0||(g&128)>0){a-=14;return}if(e<15)return C(t,n,i,o,s,c,l,u,d,f,p,m,h,g);let _=r[a++];if((_&128)>0){a-=15;return}return C(t,n,i,o,s,c,l,u,d,f,p,m,h,g,_)}}}}}function ge(){let e=r[a++],t;if(e<192)t=e-160;else switch(e){case 217:t=r[a++];break;case 218:t=_.getUint16(a),a+=2;break;case 219:t=_.getUint32(a),a+=4;break;default:throw Error(`Expected string`)}return S(t)}function _e(e){return l.copyBuffers?Uint8Array.prototype.slice.call(r,a,a+=e):r.subarray(a,a+=e)}function w(e){let t=r[a++];if(g[t]){let n;return g[t](r.subarray(a,n=a+=e),e=>{a=e;try{return x()}finally{a=n}})}else throw Error(`Unknown extension type `+t)}var ve=Array(4096);function ye(){let e=r[a++];if(e>=160&&e<192){if(e-=160,p>=a)return d.slice(a-f,(a+=e)-f);if(!(p==0&&i<180))return ce(e)}else return a--,be(x());let t=(e<<5^(e>1?_.getUint16(a):e>0?r[a]:0))&4095,n=ve[t],o=a,s=a+e-3,c,l=0;if(n&&n.bytes==e){for(;o<s;){if(c=_.getUint32(o),c!=n[l++]){o=1879048192;break}o+=4}for(s+=3;o<s;)if(c=r[o++],c!=n[l++]){o=1879048192;break}if(o===s)return a=o,n.string;s-=3,o=a}for(n=[],ve[t]=n,n.bytes=e;o<s;)c=_.getUint32(o),n.push(c),o+=4;for(s+=3;o<s;)c=r[o++],n.push(c);let u=e<16?he(e):me(e);return u==null?n.string=ce(e):n.string=u}function be(e){if(typeof e==`string`)return e;if(typeof e==`number`||typeof e==`boolean`||typeof e==`bigint`)return e.toString();if(e==null)return e+``;if(l.allowArraysInMapKeys&&Array.isArray(e)&&e.flat().every(e=>[`string`,`number`,`boolean`,`bigint`].includes(typeof e)))return e.flat().toString();throw Error(`Invalid property type for record: ${typeof e}`)}const xe=(e,t)=>{let n=x().map(be),r=e;t!==void 0&&(e=e<32?-((t<<5)+e):(t<<5)+e,n.highByte=t);let i=u[e];return i&&(i.isShared||v)&&((u.restoreStructures||=[])[e]=i),u[e]=n,n.read=ae(n,r),n.read()};g[0]=()=>{},g[0].noBuffer=!0,g[66]=e=>{let t=e.byteLength%8||8,n=BigInt(e[0]&128?e[0]-256:e[0]);for(let r=1;r<t;r++)n<<=BigInt(8),n+=BigInt(e[r]);if(e.byteLength!==t){let r=new DataView(e.buffer,e.byteOffset,e.byteLength),i=(e,t)=>{let n=t-e;if(n<=40){let n=r.getBigUint64(e);for(let i=e+8;i<t;i+=8)n<<=BigInt(64),n|=r.getBigUint64(i);return n}let a=e+(n>>4<<3),o=i(e,a),s=i(a,t);return o<<BigInt((t-a)*8)|s};n=n<<BigInt((r.byteLength-t)*8)|i(t,r.byteLength)}return n};let Se={Error,EvalError,RangeError,ReferenceError,SyntaxError,TypeError,URIError,AggregateError:typeof AggregateError==`function`?AggregateError:null};g[101]=()=>{let e=x();if(!Se[e[0]]){let t=Error(e[1],{cause:e[2]});return t.name=e[0],t}return Se[e[0]](e[1],{cause:e[2]})},g[105]=e=>{if(l.structuredClone===!1)throw Error(`Structured clone extension is disabled`);let t=_.getUint32(a-4);h||=new Map;let n=r[a],i;i=n>=144&&n<160||n==220||n==221?[]:n>=128&&n<144||n==222||n==223?new Map:(n>=199&&n<=201||n>=212&&n<=216)&&r[a+1]===115?new Set:{};let o={target:i};h.set(t,o);let s=x();if(o.used)Object.assign(i,s);else return o.target=s;if(i instanceof Map)for(let[e,t]of s.entries())i.set(e,t);if(i instanceof Set)for(let e of Array.from(s))i.add(e);return i},g[112]=e=>{if(l.structuredClone===!1)throw Error(`Structured clone extension is disabled`);let t=_.getUint32(a-4),n=h.get(t);return n.used=!0,n.target},g[115]=()=>new Set(x());const Ce=[`Int8`,`Uint8`,`Uint8Clamped`,`Int16`,`Uint16`,`Int32`,`Uint32`,`Float32`,`Float64`,`BigInt64`,`BigUint64`].map(e=>e+`Array`);let we=typeof globalThis==`object`?globalThis:window;g[116]=e=>{let t=e[0],n=Uint8Array.prototype.slice.call(e,1).buffer,r=Ce[t];if(!r){if(t===16)return n;if(t===17)return new DataView(n);throw Error(`Could not find typed array for code `+t)}return new we[r](n)},g[120]=()=>{let e=x();return new RegExp(e[0],e[1])};const Te=[];g[98]=e=>{let t=(e[0]<<24)+(e[1]<<16)+(e[2]<<8)+e[3],n=a;return a+=t-e.length,m=Te,m=[ge(),ge()],m.position0=0,m.position1=0,m.postBundlePosition=a,a=n,x()},g[255]=e=>e.length==4?new Date((e[0]*16777216+(e[1]<<16)+(e[2]<<8)+e[3])*1e3):e.length==8?new Date(((e[0]<<22)+(e[1]<<14)+(e[2]<<6)+(e[3]>>2))/1e6+((e[3]&3)*4294967296+e[4]*16777216+(e[5]<<16)+(e[6]<<8)+e[7])*1e3):e.length==12?new Date(((e[0]<<24)+(e[1]<<16)+(e[2]<<8)+e[3])/1e6+((e[4]&128?-281474976710656:0)+e[6]*1099511627776+e[7]*4294967296+e[8]*16777216+(e[9]<<16)+(e[10]<<8)+e[11])*1e3):new Date(`invalid`);function Ee(e){l&&l._onSaveState&&l._onSaveState();let t=i,n=a,o=c,g=f,ee=p,te=d,ne=s,y=h,b=m,re=new Uint8Array(r.slice(0,i)),x=u,ie=u.slice(0,u.length),ae=l,oe=v,se=e();return i=t,a=n,c=o,f=g,p=ee,d=te,s=ne,h=y,m=b,r=re,v=oe,u=x,u.splice(0,u.length,...ie),l=ae,_=new DataView(r.buffer,r.byteOffset,r.byteLength),se}function De(){r=null,h=null,u=null}const Oe=Array(147);for(let e=0;e<256;e++)Oe[e]=+(`1e`+Math.floor(45.15-e*.30103));var T=new y({useRecords:!1});T.unpack,T.unpackMultiple,T.unpack,new Uint8Array(new Float32Array(1).buffer,0,4),y.SUPPORTS_STRUCT_HOOKS=!0;const ke={gpt:{encoding:`o200k_base`,kind:`tiktoken-builtin`,openrouter:`openai/gpt-5.5`,source:{encodingJsonUrl:`https://tiktoken.pages.dev/js/o200k_base.json`},title:`GPT-5.5`},gemma:{kind:`huggingface`,openrouter:`google/gemma-4-31b-it`,source:{tokenizerConfigUrl:`https://huggingface.co/google/gemma-4-31B-it/resolve/main/tokenizer_config.json`,tokenizerJsonUrl:`https://huggingface.co/google/gemma-4-31B-it/resolve/main/tokenizer.json`},title:`Gemma 4 31B it`},qwen:{kind:`huggingface`,openrouter:`qwen/qwen3.6-27b`,source:{tokenizerConfigUrl:`https://huggingface.co/Qwen/Qwen3.6-27B/resolve/main/tokenizer_config.json`,tokenizerJsonUrl:`https://huggingface.co/Qwen/Qwen3.6-27B/resolve/main/tokenizer.json`},title:`Qwen 3.6 27B`},kimi:{kind:`tiktoken-custom`,openrouter:`moonshotai/kimi-k2.6`,source:{modelUrl:`https://huggingface.co/moonshotai/Kimi-K2.6/resolve/main/tiktoken.model`,tokenizerConfigUrl:`https://huggingface.co/moonshotai/Kimi-K2.6/resolve/main/tokenizer_config.json`,tokenizerImplementationUrl:`https://huggingface.co/moonshotai/Kimi-K2.6/resolve/main/tokenization_kimi.py`},title:`Kimi K2.6`},deepseek:{kind:`huggingface`,openrouter:`deepseek/deepseek-v4-pro`,source:{tokenizerConfigUrl:`https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/resolve/main/tokenizer_config.json`,tokenizerJsonUrl:`https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/resolve/main/tokenizer.json`},title:`DeepSeek V4 Pro`},mimo:{kind:`huggingface`,openrouter:`xiaomi/mimo-v2.5-pro`,source:{tokenizerConfigUrl:`https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro/resolve/main/tokenizer_config.json`,tokenizerJsonUrl:`https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro/resolve/main/tokenizer.json`},title:`MiMo V2.5 Pro`},sdxl:{kind:`clip-bpe`,source:{mergesUrl:`https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/resolve/main/tokenizer_2/merges.txt`,specialTokensMapUrl:`https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/resolve/main/tokenizer_2/special_tokens_map.json`,tokenizerConfigUrl:`https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/resolve/main/tokenizer_2/tokenizer_config.json`,vocabUrl:`https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/resolve/main/tokenizer_2/vocab.json`},title:`Stable Diffusion XL`},glm:{kind:`huggingface`,openrouter:`zai-org/glm-5.1`,source:{tokenizerConfigUrl:`https://huggingface.co/zai-org/GLM-5.1/resolve/main/tokenizer_config.json`,tokenizerJsonUrl:`https://huggingface.co/zai-org/GLM-5.1/resolve/main/tokenizer.json`},title:`GLM 5.1`},minimax:{kind:`huggingface`,openrouter:`minimax/minimax-m2.7`,source:{tokenizerConfigUrl:`https://huggingface.co/MiniMaxAI/MiniMax-M2.7/resolve/main/tokenizer_config.json`,tokenizerJsonUrl:`https://huggingface.co/MiniMaxAI/MiniMax-M2.7/resolve/main/tokenizer.json`},title:`MiniMax M2.7`},hy:{kind:`huggingface`,openrouter:`tencent/hy3-preview`,title:`Hy3 Preview`,source:{tokenizerConfigUrl:`https://huggingface.co/tencent/Hy3-preview/resolve/main/tokenizer_config.json`,tokenizerJsonUrl:`https://huggingface.co/tencent/Hy3-preview/resolve/main/tokenizer.json`}},step:{kind:`huggingface`,openrouter:`stepfun/step-3.7-flash`,source:{specialTokensMapUrl:`https://huggingface.co/stepfun-ai/Step-3.7-Flash/resolve/main/special_tokens_map.json`,tokenizerConfigUrl:`https://huggingface.co/stepfun-ai/Step-3.7-Flash/resolve/main/tokenizer_config.json`,tokenizerJsonUrl:`https://huggingface.co/stepfun-ai/Step-3.7-Flash/resolve/main/tokenizer.json`},title:`Step 3.7 Flash`}},E=Object.keys(ke),Ae=new y({mapsAsObjects:!1}),je=new TextDecoder,D=Object.create(null),O=new Map,Me=Object.create(null),k=new Map,A=new Map;let Ne;const Pe=(e,t)=>`${e}/${t}`,Fe=e=>Me[e]??0,Ie=e=>{let t=Fe(e)+1;return Me[e]=t,t},Le=e=>{let t=`${e}/`;for(let e of A.keys())e.startsWith(t)&&A.delete(e);for(let e of k.keys())e.startsWith(t)&&k.delete(e)},Re=e=>{let t=D[e];if(!t)throw Error(`Missing tokenizer assets for model ${JSON.stringify(e)}. Call load() first or use tokenize()/count() to auto-load it.`);return t},ze=()=>{if(!Ne)throw Error(`No tokenizer asset loader is registered for the current entry.`);return Ne},Be=(e,t)=>{let n=Re(e);if(!Object.hasOwn(n,t))throw Error(`Missing tokenizer asset ${JSON.stringify(t)} for model ${JSON.stringify(e)}. Run “bun run fetch” first.`);return n[t]},Ve=(e,t)=>Be(e,t),He=e=>{if(e.endsWith(`.msgpack`))return e;if(e.endsWith(`.json`))return`${e.slice(0,-5)}.msgpack`;throw TypeError(`Expected a MessagePack or JSON file name, got ${JSON.stringify(e)}.`)},Ue=e=>Object.hasOwn(D,e),We=e=>{Ne=e},Ge=(e,t)=>{D[e]=t,Le(e)},Ke=async e=>{await Promise.all(e.map(async e=>{if(Ue(e))return;let t=O.get(e);if(t){await t;return}let n=Fe(e),r=(async()=>{let t=await ze().load(e);Fe(e)===n&&Ge(e,t)})().finally(()=>{O.get(e)===r&&O.delete(e)});O.set(e,r),await r}))},qe=e=>{Ie(e),delete D[e],O.delete(e),Le(e)},Je=e=>{if(e){qe(e);return}for(let e of E)qe(e)},Ye=(e,t)=>{if(t.endsWith(`.json`)||t.endsWith(`.msgpack`))throw TypeError(`Cannot read structured tokenizer asset ${JSON.stringify(t)} as text.`);let n=Pe(e,t),r=A.get(n);if(r!==void 0)return r;let i=je.decode(Ve(e,t));return A.set(n,i),i},j=(e,t)=>{let n=He(t),r=Pe(e,n),i=k.get(r);if(i!==void 0)return i;let a=Ae.unpack(Ve(e,n));return k.set(r,a),a},Xe={gpt:new URL(`../gpt.msgpack.br`,import.meta.url),gemma:new URL(`../gemma.msgpack.br`,import.meta.url),qwen:new URL(`../qwen.msgpack.br`,import.meta.url),kimi:new URL(`../kimi.msgpack.br`,import.meta.url),deepseek:new URL(`../deepseek.msgpack.br`,import.meta.url),mimo:new URL(`../mimo.msgpack.br`,import.meta.url),sdxl:new URL(`../sdxl.msgpack.br`,import.meta.url),glm:new URL(`../glm.msgpack.br`,import.meta.url),minimax:new URL(`../minimax.msgpack.br`,import.meta.url),hy:new URL(`../hy.msgpack.br`,import.meta.url),step:new URL(`../step.msgpack.br`,import.meta.url)};let Ze;const Qe=()=>(Ze??=(async()=>(await import(`./decompress.js`).then(e=>t(e.default,1))).default)(),Ze),$e=async e=>{if(typeof DecompressionStream==`function`)try{let t=new Blob([e]).stream().pipeThrough(new DecompressionStream(`brotli`));return new Uint8Array(await new Response(t).arrayBuffer())}catch{}return(await Qe())(e)},et=new y({mapsAsObjects:!1}),tt=e=>{if(e instanceof Uint8Array)return e;if(e instanceof ArrayBuffer)return new Uint8Array(e);if(ArrayBuffer.isView(e))return new Uint8Array(e.buffer,e.byteOffset,e.byteLength);throw TypeError(`Expected bundled tokenizer asset contents to be binary, got ${typeof e}.`)},nt=e=>{if(!(e instanceof Map))throw TypeError(`Expected a tokenizer asset bundle to unpack into a Map.`);let t=Object.create(null);for(let[n,r]of e.entries()){if(typeof n!=`string`)throw TypeError(`Expected tokenizer asset file names to be strings, got ${typeof n}.`);t[n]=tt(r)}return t};var rt=class{async load(e){let t=await $e(await this.readCompressedBundle(e));return nt(et.unpack(t))}};We(new class extends rt{bundleUrls;constructor(e){super(),this.bundleUrls=e}async readCompressedBundle(e){let t=this.bundleUrls[e],n=await fetch(t);if(!n.ok)throw Error(`Failed to fetch tokenizer assets for model ${JSON.stringify(e)} from ${JSON.stringify(t.href)}: ${n.status} ${n.statusText}`);return new Uint8Array(await n.arrayBuffer())}}(Xe));const it=e=>{let t=[...E];e!==void 0&&(t.length=0,typeof e==`string`?t.push(e):t.push(...e));for(let e of t)if(!E.includes(e))throw Error(`Unknown model ${JSON.stringify(e)}. Supported models: ${E.join(`, `)}`);return t},at=new TextDecoder(`utf-8`,{fatal:!0}),ot=new TextEncoder,st=e=>{let t=[],n=0;for(let r of e)t.push(n),n+=r;return t},ct=e=>{if(typeof e==`string`)return e;try{return at.decode(e)}catch{throw TypeError(`Tokenizer input bytes must be valid UTF-8.`)}},lt=e=>ot.encode(e).byteLength,ut=(e,t,n)=>{if(e.length!==t.length)throw TypeError(`Expected ${e.length} token start offsets, got ${t.length}.`);let r={offsets:t.slice(1),tokens:[...e]};return n===void 0?r:{...r,processedInput:n}},dt=(e,t,n)=>{if(e.length!==t.length)throw TypeError(`Expected ${e.length} token byte lengths, got ${t.length}.`);return ut(e,st(t),n)},ft=new TextEncoder;var pt=class{#e=!1;#t;disposeState(e){}encode(e){return this.encodeWithState(ct(e),this.getState())}free(){this.#e&&=(this.disposeState(this.#t),this.#t=void 0,!1)}getState(){return this.#e||=(this.#t=this.createState(),!0),this.#t}getTokenCount(e){return this.encode(e).length}tokenize(e){let t=this.tokenizeWithState(ct(e),this.getState());return typeof e==`string`||t.processedInput===void 0?t:{...t,processedInput:ft.encode(t.processedInput)}}};const mt=/<\|startoftext\|>|<\|endoftext\|>|'d|'ll|'m|'re|'s|'t|'ve|\p{L}+|\p{N}|[^\s\p{L}\p{N}]+/gu,ht=/<\/w>$/u,gt=/\s+/gu,_t=new TextEncoder,vt=e=>{if(typeof e==`string`)return e;if(e instanceof Map){let t=e.get(`content`);if(typeof t==`string`)return t}if(e&&typeof e==`object`&&`content`in e&&typeof e.content==`string`)return e.content},yt=()=>{let e=Array.from({length:256},()=>``),t=[...Array.from({length:94},(e,t)=>t+33),...Array.from({length:12},(e,t)=>t+161),...Array.from({length:82},(e,t)=>t+174)],n=[...t],r=256;for(let e=0;e<256;e++)t.includes(e)||(t.push(e),n.push(r),r++);for(let[r,i]of t.entries())e[i]=String.fromCodePoint(n[r]);return e},bt=e=>{let t=new Map;for(let[n,r]of e.split(/\r?\n/u).filter(Boolean).filter(e=>!e.startsWith(`#`)).entries())t.set(r,n);return t},xt=e=>e.normalize(`NFC`).replaceAll(gt,` `).trim().toLowerCase(),St=e=>[...e.replace(ht,``)].length,Ct=(e,t,n)=>n===t?e:{...e,processedInput:n};var wt=class extends pt{modelId;#e=new Map;constructor(e){super(),this.modelId=e}createState(){let e=j(this.modelId,`vocab.msgpack`),t=j(this.modelId,`tokenizer_config.msgpack`),n=new Map,r=vt(t.get(`unk_token`))??`<|endoftext|>`,i=e.get(r);if(i===void 0)throw Error(`Could not find CLIP unknown token ${JSON.stringify(r)} in ${JSON.stringify(this.modelId)} vocabulary.`);for(let r of[t.get(`bos_token`),t.get(`eos_token`),t.get(`unk_token`)]){let t=vt(r);if(!t)continue;let i=e.get(t);i!==void 0&&n.set(t,i)}return{byteEncoder:yt(),mergeRanks:bt(Ye(this.modelId,`merges.txt`)),specialTokenIds:n,unknownTokenId:i,vocabulary:e}}encodeWithState(e,t){return this.tokenizeWithState(e,t).tokens}tokenizeWithState(e,t){let n=xt(e);if(!n)return Ct({offsets:[],tokens:[]},e,n);let r=[],i=[],a=0,o=0;for(let e of n.matchAll(mt)){let s=e[0],c=e.index;a+=lt(n.slice(o,c));let l=a,u=t.specialTokenIds.get(s);if(u!==void 0){r.push(u),i.push(l),a+=lt(s),o=c+s.length;continue}let d=Array.from(_t.encode(s),e=>t.byteEncoder[e]).join(``),f=0;for(let e of this.#t(d,t.mergeRanks))r.push(t.vocabulary.get(e)??t.unknownTokenId),i.push(l+f),f+=St(e);a+=lt(s),o=c+s.length}return Ct(ut(r,i),e,n)}#t(e,t){let n=this.#e.get(e);if(n)return n;let r=[...e];if(!r.length){let t=[];return this.#e.set(e,t),t}let i=[...r.slice(0,-1),`${r.at(-1)}</w>`];for(;i.length>1;){let e=-1,n=1/0;for(let r=0;r<i.length-1;r++){let a=t.get(`${i[r]} ${i[r+1]}`);a!==void 0&&a<n&&(n=a,e=r)}if(e<0)break;let r=[];for(let t=0;t<i.length;t++){if(t===e){r.push(`${i[t]}${i[t+1]}`),t++;continue}r.push(i[t])}i=r}return this.#e.set(e,i),i}},Tt=class{constructor(e){this.trie=this._build_trie(e)}_build_trie(e){let t=Object.create(null);for(let n of e){let e=t;for(let t=0;t<n.length;++t){let r=n[t];e=e[r]??=Object.create(null)}e.end=n}return t}split(e){let t=[],n=e.length,r=0,i=0;for(;i<n;){let a=this.trie,o=null,s=i;for(;s<n&&(a=a[e[s]]);)a.end&&(o=a.end),++s;o?(i>r&&t.push(e.slice(r,i)),t.push(o),i+=o.length,r=i):++i}return r<n&&t.push(e.slice(r)),t}},Et=class{constructor(e){this.content=e.content,this.id=e.id,this.single_word=e.single_word??!1,this.lstrip=e.lstrip??!1,this.rstrip=e.rstrip??!1,this.special=e.special??!1,this.normalized=e.normalized??!this.special}},Dt=(()=>{let e=[...Array.from({length:94},(e,t)=>t+33),...Array.from({length:12},(e,t)=>t+161),...Array.from({length:82},(e,t)=>t+174)],t=e.slice(),n=0;for(let r=0;r<256;++r)e.includes(r)||(e.push(r),t.push(256+n),n+=1);let r=t.map(e=>String.fromCharCode(e));return Object.fromEntries(e.map((e,t)=>[e,r[t]]))})(),Ot=(e=>Object.fromEntries(Object.entries(e).map(([e,t])=>[t,e])))(Dt),kt=`.,!?…。，、।۔،`,At=new Map([[`(?i:'s|'t|'re|'ve|'m|'ll|'d)`,`(?:'([sS]|[tT]|[rR][eE]|[vV][eE]|[mM]|[lL][lL]|[dD]))`],[`(?i:[sdmt]|ll|ve|re)`,`(?:[sS]|[dD]|[mM]|[tT]|[lL][lL]|[vV][eE]|[rR][eE])`],[`[^\\r\\n\\p{L}\\p{N}]?+`,`[^\\r\\n\\p{L}\\p{N}]?`],[`[^\\s\\p{L}\\p{N}]++`,`[^\\s\\p{L}\\p{N}]+`],[`(?>\\p{Nd}{510})`,`(?:\\p{Nd}{510})`],[`\\p{Nd}{3}+`,`(?:\\p{Nd}{3})+`],[`\\G`,``],[` ?[^(\\s|[${kt}])]+`,` ?[^\\s${kt}]+`]]),jt=`\\p{P}\\u0021-\\u002F\\u003A-\\u0040\\u005B-\\u0060\\u007B-\\u007E`,Mt=e=>e.replace(/ \./g,`.`).replace(/ \?/g,`?`).replace(/ \!/g,`!`).replace(/ ,/g,`,`).replace(/ \' /g,`'`).replace(/ n't/g,`n't`).replace(/ 'm/g,`'m`).replace(/ 's/g,`'s`).replace(/ 've/g,`'ve`).replace(/ 're/g,`'re`),Nt=(e,t=!0)=>{if(e.Regex!==void 0){let t=e.Regex.replace(/\\([#&~])/g,`$1`);t=t.replace(/\\A/g,`^`).replace(/\\z/g,`$`).replace(/\\Z/g,`(?=\\r?\\n?$)`);for(let[e,n]of At)t=t.replaceAll(e,n);try{return new RegExp(t,`gu`)}catch(e){if(!(e instanceof SyntaxError)||!e.message.toLowerCase().includes(`invalid property name`))throw e;let n=!1,r=t.replace(/(\\[pP])\{([^}=]+)\}/g,(e,t,r)=>{try{return RegExp(`\\p{${r}}`,`u`),`${t}{${r}}`}catch{return n=!0,`${t}{Script=${r}}`}});if(!n)throw e;try{return new RegExp(r,`gu`)}catch{throw e}}}else if(e.String!==void 0){let n=Pt(e.String);return new RegExp(t?n:`(${n})`,`gu`)}else return console.warn(`Unknown pattern type:`,e),null},Pt=e=>e.replace(/[.*+?^${}()|[\]\\]/g,`\\$&`),Ft=(e,t,n)=>{let r=[],i=0;for(;i<e.length;){if(r.push(e[i]),(t.get(e[i])??n)!==n){++i;continue}for(;++i<e.length&&(t.get(e[i])??n)===n;)t.get(r.at(-1))!==n&&(r[r.length-1]+=e[i])}return r},It=e=>e>=19968&&e<=40959||e>=13312&&e<=19903||e>=131072&&e<=173791||e>=173824&&e<=177983||e>=177984&&e<=178207||e>=178208&&e<=183983||e>=63744&&e<=64255||e>=194560&&e<=195103,Lt=e=>Number.isInteger(e)||typeof e==`bigint`,Rt=e=>{let t=0;for(let n of e)++t;return t},zt=e=>Ht(e.toLowerCase()),M=(...e)=>Array.prototype.concat.apply([],e),Bt=e=>new Map(Object.entries(e)),Vt=(e,t)=>{let n=[],r=0;for(let i of e.matchAll(t)){let t=i[0];r<i.index&&n.push(e.slice(r,i.index)),t.length>0&&n.push(t),r=i.index+t.length}return r<e.length&&n.push(e.slice(r)),n},Ht=e=>e.replace(/\p{M}/gu,``),Ut=(e,t,n=[])=>{if(!e||Array.isArray(e)||typeof e!=`object`)return`${t} must be a valid object`;for(let r of n)if(!(r in e))return`${t} must contain a "${r}" property`;return null},Wt=e=>e.match(/\S+/g)||[],N=class{constructor(){let e=function(...t){return e._call(...t)};return Object.setPrototypeOf(e,new.target.prototype)}},P=class extends N{constructor(e){super(),this.config=e}_call(e){return this.normalize(e)}},Gt=class extends P{tokenize_chinese_chars(e){let t=[];for(let n=0;n<e.length;++n){let r=e[n];It(r.charCodeAt(0))?(t.push(` `),t.push(r),t.push(` `)):t.push(r)}return t.join(``)}strip_accents(e){return e.normalize(`NFD`).replace(/\p{Mn}/gu,``)}is_control(e){switch(e){case`	`:case`
-`:case`\r`:return!1;default:return/^\p{Cc}|\p{Cf}|\p{Co}|\p{Cs}$/u.test(e)}}clean_text(e){let t=[];for(let n of e){let e=n.charCodeAt(0);e===0||e===65533||this.is_control(n)||(/^\s$/.test(n)?t.push(` `):t.push(n))}return t.join(``)}normalize(e){return this.config.clean_text&&(e=this.clean_text(e)),this.config.handle_chinese_chars&&(e=this.tokenize_chinese_chars(e)),this.config.lowercase?(e=e.toLowerCase(),this.config.strip_accents!==!1&&(e=this.strip_accents(e))):this.config.strip_accents&&(e=this.strip_accents(e)),e}},Kt=class extends P{constructor(e){super(e),this.charsmap=e.precompiled_charsmap??null}normalize(e){return e=e.replace(/[\u0001-\u0008\u000B\u000E-\u001F\u007F\u008F\u009F]/gm,``),e=e.replace(/[\u0009\u000A\u000C\u000D\u00A0\u1680\u2000-\u200F\u2028\u2029\u202F\u205F\u2581\u3000\uFEFF\uFFFD]/gm,` `),e=e.includes(`～`)?e.split(`～`).map(e=>e.normalize(`NFKC`)).join(`～`):e.normalize(`NFKC`),e}},qt=class extends P{constructor(e){super(e),this.normalizers=(e.normalizers??[]).map(e=>an(e))}normalize(e){return this.normalizers.reduce((e,t)=>t?t.normalize(e):e,e)}},Jt=class extends P{normalize(e){let t=Nt(this.config.pattern??{});return t===null?e:e.replaceAll(t,this.config.content??``)}},F=class extends P{constructor(){super(...arguments),this.form=`NFC`}normalize(e){return e=e.normalize(this.form),e}},Yt=class extends F{constructor(){super(...arguments),this.form=`NFC`}},Xt=class extends F{constructor(){super(...arguments),this.form=`NFD`}},Zt=class extends F{constructor(){super(...arguments),this.form=`NFKC`}},Qt=class extends F{constructor(){super(...arguments),this.form=`NFKD`}},$t=class extends P{normalize(e){return this.config.strip_left&&this.config.strip_right?e=e.trim():(this.config.strip_left&&(e=e.trimStart()),this.config.strip_right&&(e=e.trimEnd())),e}},en=class extends P{normalize(e){return Ht(e)}},tn=class extends P{normalize(e){return e.toLowerCase()}},nn=class extends P{normalize(e){return e=this.config.prepend+e,e}};function rn(e){if(e===null)return null;switch(e.type){case`BertNormalizer`:return new Gt(e);case`Precompiled`:return new Kt(e);case`Sequence`:return new qt(e);case`Replace`:return new Jt(e);case`NFC`:return new Yt(e);case`NFD`:return new Xt(e);case`NFKC`:return new Zt(e);case`NFKD`:return new Qt(e);case`Strip`:return new $t(e);case`StripAccents`:return new en(e);case`Lowercase`:return new tn(e);case`Prepend`:return new nn(e);default:throw Error(`Unknown Normalizer type: ${e.type}`)}}var an=rn,I=class extends N{pre_tokenize(e,t){return(Array.isArray(e)?e.map(e=>this.pre_tokenize_text(e,t)):this.pre_tokenize_text(e,t)).flat()}_call(e,t){return this.pre_tokenize(e,t)}},on=class extends I{constructor(e){super(),this.config=e,this.add_prefix_space=this.config.add_prefix_space??!1,this.trim_offsets=this.config.trim_offsets??!1,this.use_regex=this.config.use_regex??!0,this.pattern=/'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu,this.byte_encoder=Dt,this.text_encoder=new TextEncoder}pre_tokenize_text(e,t){return this.add_prefix_space&&!e.startsWith(` `)&&(e=` `+e),(this.use_regex?e.match(this.pattern)||[]:[e]).map(e=>Array.from(this.text_encoder.encode(e),e=>this.byte_encoder[e]).join(``))}},sn=class extends I{pre_tokenize_text(e,t){return e.match(/\w+|[^\w\s]+/g)||[]}},cn=class extends I{constructor(e){super(),this.replacement=e.replacement??`▁`,this.str_rep=e.str_rep||this.replacement,this.prepend_scheme=e.prepend_scheme??`always`}pre_tokenize_text(e,t){let{section_index:n=void 0}=t??{},r=e.replaceAll(` `,this.str_rep);return!r.startsWith(this.replacement)&&(this.prepend_scheme===`always`||this.prepend_scheme===`first`&&n===0)&&(r=this.str_rep+r),[r]}},ln=class extends I{constructor(e){super(),this.config=e,this.pattern=Nt(this.config.pattern??{},this.config.invert??!0)}pre_tokenize_text(e){return this.pattern===null?[]:this.config.invert?e.match(this.pattern)||[]:this.config.behavior?.toLowerCase()===`removed`?e.split(this.pattern).filter(e=>e):Vt(e,this.pattern)}},un=class extends I{constructor(e){super(),this.config=e,this.pattern=RegExp(`[^${jt}]+|[${jt}]+`,`gu`)}pre_tokenize_text(e){return e.match(this.pattern)||[]}},dn=class extends I{constructor(e){super(),this.config=e;let t=`[^\\d]+|\\d${this.config.individual_digits?``:`+`}`;this.pattern=new RegExp(t,`gu`)}pre_tokenize_text(e){return e.match(this.pattern)||[]}},fn=class extends I{constructor(){super(),this.pattern=RegExp(`[^\\s${jt}]+|[${jt}]`,`gu`)}pre_tokenize_text(e,t){return e.trim().match(this.pattern)||[]}},pn=class extends I{constructor(e){super(),this.config=e,this.pattern=Nt(this.config.pattern??{}),this.content=this.config.content??``}pre_tokenize_text(e){return this.pattern===null?[e]:[e.replaceAll(this.pattern,this.config.content??``)]}},mn=class extends I{constructor(e){super(),this.tokenizers=(e.pretokenizers??[]).map(e=>vn(e))}pre_tokenize_text(e,t){return this.tokenizers.reduce((e,n)=>n?n.pre_tokenize(e,t):e,[e])}},hn=class extends I{pre_tokenize_text(e){return Wt(e)}},gn=class extends I{constructor(e){super(),this.config=e,this._length=e.length}pre_tokenize_text(e){let t=[];for(let n=0;n<e.length;n+=this._length)t.push(e.slice(n,n+this._length));return t}};function _n(e){if(e===null)return null;switch(e.type){case`BertPreTokenizer`:return new fn;case`Sequence`:return new mn(e);case`Whitespace`:return new sn;case`WhitespaceSplit`:return new hn;case`Metaspace`:return new cn(e);case`ByteLevel`:return new on(e);case`Split`:return new ln(e);case`Punctuation`:return new un(e);case`Digits`:return new dn(e);case`Replace`:return new pn(e);case`FixedLength`:return new gn(e);default:throw Error(`Unknown PreTokenizer type: ${e.type}`)}}var vn=_n,L=class extends N{constructor(e){super(),this.config=e,this.vocab=[],this.tokens_to_ids=new Map,this.unk_token_id=void 0,this.unk_token=void 0,this.end_of_word_suffix=void 0,this.fuse_unk=this.config.fuse_unk??!1}_call(e){let t=this.encode(e);return this.fuse_unk&&(t=Ft(t,this.tokens_to_ids,this.unk_token_id)),t}},yn=class extends L{constructor(e){super(e),this.max_input_chars_per_word=100,this.tokens_to_ids=Bt(e.vocab),this.unk_token_id=this.tokens_to_ids.get(e.unk_token),this.unk_token=e.unk_token,this.max_input_chars_per_word=e.max_input_chars_per_word??100,this.vocab=Array(this.tokens_to_ids.size);for(let[e,t]of this.tokens_to_ids)this.vocab[t]=e}encode(e){let t=[];for(let n of e){let e=[...n];if(e.length>this.max_input_chars_per_word){t.push(this.unk_token);continue}let r=!1,i=0,a=[];for(;i<e.length;){let t=e.length,n=null;for(;i<t;){let r=e.slice(i,t).join(``);if(i>0&&(r=this.config.continuing_subword_prefix+r),this.tokens_to_ids.has(r)){n=r;break}--t}if(n===null){r=!0;break}a.push(n),i=t}r?t.push(this.unk_token):t.push(...a)}return t}},bn=class e{constructor(e,t){this.is_leaf=e,this.children=t}static default(){return new e(!1,new Map)}},xn=class{constructor(){this.root=bn.default()}extend(e){for(let t of e)this.push(t)}push(e){let t=this.root;for(let n of e){let e=t.children.get(n);e===void 0&&(e=bn.default(),t.children.set(n,e)),t=e}t.is_leaf=!0}*common_prefix_search(e){let t=this.root;if(t===void 0)return;let n=``;for(let r of e){if(n+=r,t=t.children.get(r),t===void 0)return;t.is_leaf&&(yield n)}}},Sn=class e{constructor(e,t,n,r,i){this.token_id=e,this.node_id=t,this.pos=n,this.length=r,this.score=i,this.prev=null,this.backtrace_score=0}clone(){let t=new e(this.token_id,this.node_id,this.pos,this.length,this.score);return t.prev=this.prev,t.backtrace_score=this.backtrace_score,t}},Cn=class{constructor(e,t,n){this.chars=Array.from(e),this.len=this.chars.length,this.bos_token_id=t,this.eos_token_id=n,this.nodes=[],this.begin_nodes=Array.from({length:this.len+1},()=>[]),this.end_nodes=Array.from({length:this.len+1},()=>[]);let r=new Sn(this.bos_token_id??0,0,0,0,0),i=new Sn(this.eos_token_id??0,1,this.len,0,0);this.nodes.push(r.clone()),this.nodes.push(i.clone()),this.begin_nodes[this.len].push(i),this.end_nodes[0].push(r)}insert(e,t,n,r){let i=this.nodes.length,a=new Sn(r,i,e,t,n);this.begin_nodes[e].push(a),this.end_nodes[e+t].push(a),this.nodes.push(a)}viterbi(){let e=this.len,t=0;for(;t<=e;){if(this.begin_nodes[t].length==0)return[];for(let e of this.begin_nodes[t]){e.prev=null;let n=0,r=null;for(let i of this.end_nodes[t]){let t=i.backtrace_score+e.score;(r===null||t>n)&&(r=i.clone(),n=t)}if(r!==null)e.prev=r,e.backtrace_score=n;else return[]}++t}let n=[],r=this.begin_nodes[e][0].prev;if(r===null)return[];let i=r.clone();for(;i.prev!==null;)n.push(i.clone()),i=i.clone().prev.clone();return n.reverse(),n}piece(e){return this.chars.slice(e.pos,e.pos+e.length).join(``)}tokens(){return this.viterbi().map(e=>this.piece(e))}token_ids(){return this.viterbi().map(e=>e.token_id)}};function wn(e){if(e.length===0)throw Error(`Array must not be empty`);let t=e[0],n=0;for(let r=1;r<e.length;++r)e[r]<t&&(t=e[r],n=r);return[t,n]}var Tn=class extends L{constructor(e,t){super(e);let n=e.vocab.length;this.vocab=Array(n),this.scores=Array(n);for(let t=0;t<n;++t)[this.vocab[t],this.scores[t]]=e.vocab[t];this.unk_token_id=e.unk_id,this.unk_token=this.vocab[e.unk_id],this.tokens_to_ids=new Map(this.vocab.map((e,t)=>[e,t])),this.bos_token=` `,this.bos_token_id=this.tokens_to_ids.get(this.bos_token),this.eos_token=t,this.eos_token_id=this.tokens_to_ids.get(this.eos_token),this.unk_token=this.vocab[this.unk_token_id],this.min_score=wn(this.scores)[0],this.unk_score=this.min_score-10,this.scores[this.unk_token_id]=this.unk_score,this.trie=new xn,this.trie.extend(this.vocab),this.fuse_unk=!0}populate_nodes(e){let t=e.chars,n=0;for(;n<t.length;){let r=!1,i=[],a=t.slice(n).join(``),o=this.trie.common_prefix_search(a);for(let t of o){i.push(t);let a=this.tokens_to_ids.get(t),o=this.scores[a],s=Rt(t);e.insert(n,s,o,a),!r&&s===1&&(r=!0)}r||e.insert(n,1,this.unk_score,this.unk_token_id),n+=1}}tokenize(e){let t=new Cn(e,this.bos_token_id,this.eos_token_id);return this.populate_nodes(t),t.tokens()}encode(e){let t=[];for(let n of e){let e=this.tokenize(n);t.push(...e)}return t}},En=class{constructor(e=(e,t)=>e>t,t=1/0){this._heap=[],this._comparator=e,this._max_size=t}get size(){return this._heap.length}is_empty(){return this.size===0}peek(){return this._heap[0]}push(...e){return this.extend(e)}extend(e){for(let t of e)if(this.size<this._max_size)this._heap.push(t),this._sift_up();else{let e=this._smallest();this._comparator(t,this._heap[e])&&(this._heap[e]=t,this._sift_up_from(e))}return this.size}pop(){let e=this.peek(),t=this.size-1;return t>0&&this._swap(0,t),this._heap.pop(),this._sift_down(),e}replace(e){let t=this.peek();return this._heap[0]=e,this._sift_down(),t}_parent(e){return(e+1>>>1)-1}_left(e){return(e<<1)+1}_right(e){return e+1<<1}_greater(e,t){return this._comparator(this._heap[e],this._heap[t])}_swap(e,t){let n=this._heap[e];this._heap[e]=this._heap[t],this._heap[t]=n}_sift_up(){this._sift_up_from(this.size-1)}_sift_up_from(e){for(;e>0&&this._greater(e,this._parent(e));)this._swap(e,this._parent(e)),e=this._parent(e)}_sift_down(){let e=0;for(;this._left(e)<this.size&&this._greater(this._left(e),e)||this._right(e)<this.size&&this._greater(this._right(e),e);){let t=this._right(e)<this.size&&this._greater(this._right(e),this._left(e))?this._right(e):this._left(e);this._swap(e,t),e=t}}_smallest(){return 2**Math.floor(Math.log2(this.size))-1}},Dn=class{constructor(e){this.capacity=e,this.cache=new Map}get(e){if(!this.cache.has(e))return;let t=this.cache.get(e);return this.cache.delete(e),this.cache.set(e,t),t}put(e,t){this.cache.has(e)&&this.cache.delete(e),this.cache.set(e,t),this.cache.size>this.capacity&&this.cache.delete(this.cache.keys().next().value)}clear(){this.cache.clear()}},On=class extends L{constructor(e){super(e),this.tokens_to_ids=Bt(e.vocab),this.unk_token_id=this.tokens_to_ids.get(e.unk_token),this.unk_token=e.unk_token,this.vocab=Array(this.tokens_to_ids.size);for(let[e,t]of this.tokens_to_ids)this.vocab[t]=e;let t=Array.isArray(e.merges[0]);this.merges=t?e.merges:e.merges.map(e=>e.split(` `,2)),this.bpe_ranks=new Map(this.merges.map((e,t)=>[JSON.stringify(e),t])),this.end_of_word_suffix=e.end_of_word_suffix,this.continuing_subword_suffix=e.continuing_subword_suffix??null,this.byte_fallback=this.config.byte_fallback??!1,this.byte_fallback&&(this.text_encoder=new TextEncoder),this.ignore_merges=this.config.ignore_merges??!1,this.max_length_to_cache=256,this.cache_capacity=1e4,this.cache=new Dn(this.cache_capacity)}clear_cache(){this.cache.clear()}bpe(e){if(e.length===0)return[];let t=this.cache.get(e);if(t!==void 0)return t;let n=Array.from(e);this.end_of_word_suffix&&(n[n.length-1]+=this.end_of_word_suffix);let r=[];if(n.length>1){let e=new En((e,t)=>e.score<t.score),t={token:n[0],bias:0,prev:null,next:null},i=t;for(let t=1;t<n.length;++t){let r={bias:t/n.length,token:n[t],prev:i,next:null};i.next=r,this.add_node(e,i),i=r}for(;!e.is_empty();){let n=e.pop();if(n.deleted||!n.next||n.next.deleted)continue;if(n.deleted=!0,n.next.deleted=!0,n.prev){let e={...n.prev};n.prev.deleted=!0,n.prev=e,e.prev?e.prev.next=e:t=e}let r={token:n.token+n.next.token,bias:n.bias,prev:n.prev,next:n.next.next};r.prev?(r.prev.next=r,this.add_node(e,r.prev)):t=r,r.next&&(r.next.prev=r,this.add_node(e,r))}for(let e=t;e!==null;e=e.next)r.push(e.token)}else r=n;if(this.continuing_subword_suffix)for(let e=0;e<r.length-1;++e)r[e]+=this.continuing_subword_suffix;return e.length<this.max_length_to_cache&&this.cache.put(e,r),r}add_node(e,t){let n=this.bpe_ranks.get(JSON.stringify([t.token,t.next.token]));n!==void 0&&(t.score=n+t.bias,e.push(t))}encode(e){let t=[];for(let n of e){if(this.ignore_merges&&this.tokens_to_ids.has(n)){t.push(n);continue}let e=this.bpe(n);for(let n of e)if(this.tokens_to_ids.has(n))t.push(n);else if(this.byte_fallback){let e=Array.from(this.text_encoder.encode(n)).map(e=>`<0x${e.toString(16).toUpperCase().padStart(2,`0`)}>`);e.every(e=>this.tokens_to_ids.has(e))?t.push(...e):this.unk_token!=null&&t.push(this.unk_token)}else this.unk_token!=null&&t.push(this.unk_token)}return t}},kn=class extends L{constructor(e,t){super(e);let n=e.vocab;this.tokens_to_ids=Bt(t.target_lang?n[t.target_lang]:n),this.bos_token=t.bos_token,this.bos_token_id=this.tokens_to_ids.get(this.bos_token),this.eos_token=t.eos_token,this.eos_token_id=this.tokens_to_ids.get(this.eos_token),this.pad_token=t.pad_token,this.pad_token_id=this.tokens_to_ids.get(this.pad_token),this.unk_token=t.unk_token,this.unk_token_id=this.tokens_to_ids.get(this.unk_token),this.vocab=Array(this.tokens_to_ids.size);for(let[e,t]of this.tokens_to_ids)this.vocab[t]=e}encode(e){return e}};function An(e,t){switch(e.type){case`WordPiece`:return new yn(e);case`Unigram`:return new Tn(e,t.eos_token);case`BPE`:return new On(e);default:if(e.vocab)return Array.isArray(e.vocab)?new Tn(e,t.eos_token):Object.hasOwn(e,`continuing_subword_prefix`)&&Object.hasOwn(e,`unk_token`)?Object.hasOwn(e,`merges`)?new On(e):new yn(e):new kn(e,{target_lang:t.target_lang,bos_token:t.bos_token,eos_token:t.eos_token,pad_token:t.pad_token,unk_token:t.unk_token});throw Error(`Unknown TokenizerModel type: ${e?.type}`)}}var jn=An,R=class extends N{constructor(e){super(),this.config=e}_call(e,...t){return this.post_process(e,...t)}},Mn=class extends R{post_process(e,t=null,n=!0){let r=t===null?this.config.single:this.config.pair,i=[],a=[];for(let o of r)`SpecialToken`in o?n&&(i.push(o.SpecialToken.id),a.push(o.SpecialToken.type_id)):`Sequence`in o&&(o.Sequence.id===`A`?(i=M(i,e),a=M(a,Array(e.length).fill(o.Sequence.type_id))):o.Sequence.id===`B`&&(i=M(i,t),a=M(a,Array(t.length).fill(o.Sequence.type_id))));return{tokens:i,token_type_ids:a}}},Nn=class extends R{post_process(e,t=null){return{tokens:e,tokens_pair:t}}},Pn=class extends R{constructor(e){super(e),this.sep=e.sep,this.cls=e.cls}post_process(e,t=null,n=!0){n&&(e=M([this.cls[0]],e,[this.sep[0]]));let r=Array(e.length).fill(0);if(t){let i=[],a=n?[this.sep[0]]:[];e=M(e,i,t,a),r=M(r,Array(t.length+i.length+a.length).fill(1))}return{tokens:e,token_type_ids:r}}},Fn=class extends R{constructor(e){super(e),this.sep=e.sep,this.cls=e.cls}post_process(e,t,n=!0){n&&(e=M([this.cls[0]],e,[this.sep[0]]));let r=Array(e.length).fill(0);if(t){let i=n?[this.sep[0]]:[],a=n?[this.sep[0]]:[];e=M(e,i,t,a),r=M(r,Array(t.length+i.length+a.length).fill(1))}return{tokens:e,token_type_ids:r}}},In=class extends R{constructor(e){super(e),this.processors=(e.processors??[]).map(e=>Rn(e))}post_process(e,t=null,n=!0){let r={tokens:e,tokens_pair:t};for(let e of this.processors)r=e.post_process(r.tokens,r.tokens_pair,n);return r}};function Ln(e){if(e===null)return null;switch(e.type){case`TemplateProcessing`:return new Mn(e);case`ByteLevel`:return new Nn(e);case`BertProcessing`:return new Pn(e);case`RobertaProcessing`:return new Fn(e);case`Sequence`:return new In(e);default:throw Error(`Unknown PostProcessor type: ${e.type}`)}}var Rn=Ln,z=class extends N{constructor(e){super(),this.config=e,this.added_tokens=[],this.end_of_word_suffix=null,this.trim_offsets=`trim_offsets`in e?e.trim_offsets:!1}_call(e){return this.decode(e)}decode(e){return this.decode_chain(e).join(``)}},zn=class extends z{constructor(e){super(e),this.byte_decoder=Ot,this.text_decoder=new TextDecoder(`utf-8`,{fatal:!1,ignoreBOM:!0}),this.end_of_word_suffix=null}convert_tokens_to_string(e){let t=e.join(``),n=new Uint8Array([...t].map(e=>this.byte_decoder[e]));return this.text_decoder.decode(n)}decode_chain(e){let t=[],n=[];for(let r of e)this.added_tokens.find(e=>e.content===r)===void 0?n.push(r):(n.length>0&&(t.push(this.convert_tokens_to_string(n)),n=[]),t.push(r));return n.length>0&&t.push(this.convert_tokens_to_string(n)),t}},Bn=class extends z{constructor(e){super(e),this.cleanup=e.cleanup}decode_chain(e){return e.map((e,t)=>{if(t!==0){let t=this.config.prefix;e=t&&e.startsWith(t)?e.replace(t,``):` `+e}return this.cleanup&&(e=Mt(e)),e})}},Vn=class extends z{constructor(e){super(e),this.replacement=e.replacement??`▁`}decode_chain(e){let t=[];for(let n=0;n<e.length;++n){let r=e[n].replaceAll(this.replacement,` `);n==0&&r.startsWith(` `)&&(r=r.substring(1)),t.push(r)}return t}},Hn=class extends z{constructor(e){super(e),this.suffix=e.suffix??``}decode_chain(e){return e.map((t,n)=>t.replaceAll(this.suffix,n===e.length-1?``:` `))}},Un=class extends z{constructor(e){super(e),this.pad_token=e.pad_token??``,this.word_delimiter_token=e.word_delimiter_token??``,this.cleanup=e.cleanup}convert_tokens_to_string(e){if(e.length===0)return``;let t=[e[0]];for(let n=1;n<e.length;++n)e[n]!==t.at(-1)&&t.push(e[n]);let n=t.filter(e=>e!==this.pad_token).join(``);return this.cleanup&&(n=Mt(n).replaceAll(this.word_delimiter_token,` `).trim()),n}decode_chain(e){return[this.convert_tokens_to_string(e)]}},Wn=class extends z{constructor(e){super(e),this.decoders=(e.decoders??[]).map(e=>Xn(e))}decode_chain(e){return this.decoders.reduce((e,t)=>t.decode_chain(e),e)}},Gn=class extends z{decode_chain(e){let t=Nt(this.config.pattern),n=this.config.content??``;return t===null?e:e.map(e=>e.replaceAll(t,n))}},Kn=class extends z{decode_chain(e){return[e.join(``)]}},qn=class extends z{constructor(e){super(e),this.content=e.content??``,this.start=e.start??0,this.stop=e.stop??0}decode_chain(e){return e.map(e=>{let t=0;for(let n=0;n<this.start&&e[n]===this.content;++n)t=n+1;let n=e.length;for(let t=0;t<this.stop;++t){let r=e.length-t-1;if(e[r]===this.content){n=r;continue}else break}return e.slice(t,n)})}},Jn=class extends z{constructor(e){super(e),this.text_decoder=new TextDecoder}decode_chain(e){let t=[],n=[];for(let r of e){let e=null;if(r.length===6&&r.startsWith(`<0x`)&&r.endsWith(`>`)){let t=parseInt(r.slice(3,5),16);isNaN(t)||(e=t)}if(e!==null)n.push(e);else{if(n.length>0){let e=this.text_decoder.decode(Uint8Array.from(n));t.push(e),n=[]}t.push(r)}}if(n.length>0){let e=this.text_decoder.decode(Uint8Array.from(n));t.push(e),n=[]}return t}};function Yn(e){if(e===null)return null;switch(e.type){case`ByteLevel`:return new zn(e);case`WordPiece`:return new Bn(e);case`Metaspace`:return new Vn(e);case`BPEDecoder`:return new Hn(e);case`CTC`:return new Un(e);case`Sequence`:return new Wn(e);case`Replace`:return new Gn(e);case`Fuse`:return new Kn(e);case`Strip`:return new qn(e);case`ByteFallback`:return new Jn(e);default:throw Error(`Unknown Decoder type: ${e.type}`)}}var Xn=Yn,Zn=class{constructor(e,t){let n=Ut(e,`Tokenizer`,[`model`,`decoder`,`post_processor`,`pre_tokenizer`,`normalizer`]);if(n)throw Error(n);let r=Ut(t,`Config`);if(r)throw Error(r);this.tokenizer=e,this.config=t,this.normalizer=an(this.tokenizer.normalizer),this.pre_tokenizer=vn(this.tokenizer.pre_tokenizer),this.model=jn(this.tokenizer.model,this.config),this.post_processor=Rn(this.tokenizer.post_processor),this.decoder=Xn(this.tokenizer.decoder),this.special_tokens=[],this.all_special_ids=[],this.added_tokens=[];let i=[],a=[];this.added_tokens_map=new Map;for(let e of this.tokenizer.added_tokens){let t=new Et(e);if(this.added_tokens.push(t),this.model.tokens_to_ids.set(t.content,t.id),this.model.vocab[t.id]=t.content,t.special&&(this.special_tokens.push(t.content),this.all_special_ids.push(t.id)),this.added_tokens_map.set(t.content,t),t.normalized&&this.normalizer!==null){let e=this.normalizer(t.content);a.push(e),this.added_tokens_map.set(e,t)}else i.push(t.content)}(this.config.additional_special_tokens??[]).forEach(e=>{this.special_tokens.includes(e)||this.special_tokens.push(e)}),this.decoder&&(this.decoder.added_tokens=this.added_tokens,this.decoder.end_of_word_suffix=this.model.end_of_word_suffix),this.splitter_unnormalized=new Tt(i),this.splitter_normalized=new Tt(a),this.remove_space=this.config.remove_space,this.clean_up_tokenization_spaces=this.config.clean_up_tokenization_spaces??!0,this.do_lowercase_and_remove_accent=this.config.do_lowercase_and_remove_accent??!1}encode(e,{text_pair:t=null,add_special_tokens:n=!0,return_token_type_ids:r=null}={}){let{tokens:i,token_type_ids:a}=this.tokenize_helper(e,{text_pair:t,add_special_tokens:n}),o=i.map(e=>this.added_tokens_map.get(e)?.id??this.model.tokens_to_ids.get(e)??this.model.unk_token_id),s={ids:o,tokens:i,attention_mask:Array(o.length).fill(1)};return r&&a&&(s.token_type_ids=a),s}decode(e,t={}){if(!Array.isArray(e)||e.length===0||!Lt(e[0]))throw Error(`token_ids must be a non-empty array of integers.`);let n=e.map(e=>this.model.vocab[Number(e)]??this.model.unk_token);t.skip_special_tokens&&(n=n.filter(e=>!this.special_tokens.includes(e)));let r=this.decoder?this.decoder(n):n.join(` `);return this.decoder&&this.decoder.end_of_word_suffix&&(r=r.replaceAll(this.decoder.end_of_word_suffix,` `),t.skip_special_tokens&&(r=r.trim())),(t.clean_up_tokenization_spaces??this.clean_up_tokenization_spaces)&&(r=Mt(r)),r}tokenize(e,{text_pair:t=null,add_special_tokens:n=!1}={}){return this.tokenize_helper(e,{text_pair:t,add_special_tokens:n}).tokens}encode_text(e){if(e===null)return null;let t=this.splitter_unnormalized.split(e);return t.forEach((e,n)=>{let r=this.added_tokens_map.get(e);r&&(r.lstrip&&n>0&&(t[n-1]=t[n-1].trimEnd()),r.rstrip&&n<t.length-1&&(t[n+1]=t[n+1].trimStart()))}),t.flatMap((e,t)=>{if(e.length===0)return[];if(this.added_tokens_map.has(e))return[e];if(this.remove_space===!0&&(e=e.trim().split(/\s+/).join(` `)),this.do_lowercase_and_remove_accent&&(e=zt(e)),this.normalizer!==null&&(e=this.normalizer(e)),e.length===0)return[];let n=this.splitter_normalized.split(e);return n.forEach((e,t)=>{let r=this.added_tokens_map.get(e);r&&(r.lstrip&&t>0&&(n[t-1]=n[t-1].trimEnd()),r.rstrip&&t<n.length-1&&(n[t+1]=n[t+1].trimStart()))}),n.flatMap(e=>{if(e.length===0)return[];if(this.added_tokens_map.has(e))return[e];let n=this.pre_tokenizer===null?[e]:this.pre_tokenizer(e,{section_index:t});return this.model(n)})})}tokenize_helper(e,{text_pair:t=null,add_special_tokens:n=!0}){let r=this.encode_text(e),i=this.encode_text(t||null);return this.post_processor?this.post_processor(r,i,n):{tokens:M(r??[],i??[])}}token_to_id(e){return this.model.tokens_to_ids.get(e)}id_to_token(e){return this.model.vocab[e]}get_added_tokens_decoder(){let e=new Map;for(let t of this.added_tokens)e.set(t.id,t);return e}get_vocab(e=!0){let t=new Map;for(let n=0;n<this.model.vocab.length;++n){let r=this.model.vocab[n];(e||!this.added_tokens_map.has(r))&&t.set(r,n)}return t}};const Qn=(e,t)=>{if(!e.has(t))throw Error(`Missing required key ${JSON.stringify(t)} in structured data.`);return e.get(t)},B=e=>{if(Array.isArray(e))return e.map(e=>B(e));if(e instanceof Map){let t=Object.create(null);for(let[n,r]of e.entries()){if(typeof n!=`string`)throw TypeError(`Expected string structured-data key, got ${typeof n}.`);t[n]=B(r)}return t}return e},$n=Zn,er={add_special_tokens:!1},tr={clean_up_tokenization_spaces:!1,skip_special_tokens:!1};var nr=class extends pt{modelId;constructor(e){super(),this.modelId=e}createState(){return{tokenizer:new $n(B(j(this.modelId,`tokenizer.msgpack`)),B(j(this.modelId,`tokenizer_config.msgpack`)))}}encodeWithState(e,t){return[...t.tokenizer.encode(e,er).ids]}tokenizeWithState(e,t){let n=this.encodeWithState(e,t);return dt(n,n.map(e=>lt(t.tokenizer.decode([e],tr))))}},rr=e({Tiktoken:()=>Sr,__wbg_parse_def2e24ef1252aff:()=>Cr,__wbg_set_wasm:()=>ir,__wbg_stringify_f7ed6987935b4a24:()=>wr,__wbindgen_error_new:()=>Tr,__wbindgen_is_undefined:()=>Er,__wbindgen_object_drop_ref:()=>Dr,__wbindgen_string_get:()=>Or,__wbindgen_throw:()=>kr,encoding_for_model:()=>yr,get_encoding:()=>vr,get_encoding_name_for_model:()=>br});let V;function ir(e){V=e}let ar=new(typeof TextDecoder>`u`?(0,module.require)(`util`).TextDecoder:TextDecoder)(`utf-8`,{ignoreBOM:!0,fatal:!0});ar.decode();let H=null;function U(){return(H===null||H.byteLength===0)&&(H=new Uint8Array(V.memory.buffer)),H}function W(e,t){return e>>>=0,ar.decode(U().subarray(e,e+t))}const G=Array(128).fill(void 0);G.push(void 0,null,!0,!1);let K=G.length;function q(e){K===G.length&&G.push(G.length+1);let t=K;return K=G[t],G[t]=e,t}function or(e,t){try{return e.apply(this,t)}catch(e){V.__wbindgen_export_0(q(e))}}function sr(e){return G[e]}function cr(e){e<132||(G[e]=K,K=e)}function J(e){let t=sr(e);return cr(e),t}let Y=0,lr=new(typeof TextEncoder>`u`?(0,module.require)(`util`).TextEncoder:TextEncoder)(`utf-8`);const ur=typeof lr.encodeInto==`function`?function(e,t){return lr.encodeInto(e,t)}:function(e,t){let n=lr.encode(e);return t.set(n),{read:e.length,written:n.length}};function X(e,t,n){if(n===void 0){let n=lr.encode(e),r=t(n.length,1)>>>0;return U().subarray(r,r+n.length).set(n),Y=n.length,r}let r=e.length,i=t(r,1)>>>0,a=U(),o=0;for(;o<r;o++){let t=e.charCodeAt(o);if(t>127)break;a[i+o]=t}if(o!==r){o!==0&&(e=e.slice(o)),i=n(i,r,r=o+e.length*3,1)>>>0;let t=U().subarray(i+o,i+r),a=ur(e,t);o+=a.written,i=n(i,r,o,1)>>>0}return Y=o,i}function dr(e){return e==null}let Z=null;function Q(){return(Z===null||Z.buffer.detached===!0||Z.buffer.detached===void 0&&Z.buffer!==V.memory.buffer)&&(Z=new DataView(V.memory.buffer)),Z}let fr=null;function pr(){return(fr===null||fr.byteLength===0)&&(fr=new Uint32Array(V.memory.buffer)),fr}function mr(e,t){return e>>>=0,pr().subarray(e/4,e/4+t)}function hr(e,t){let n=t(e.length*1,1)>>>0;return U().set(e,n/1),Y=e.length,n}function gr(e,t){let n=t(e.length*4,4)>>>0;return pr().set(e,n/4),Y=e.length,n}function _r(e,t){return e>>>=0,U().subarray(e/1,e/1+t)}function vr(e,t){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);try{let i=V.__wbindgen_add_to_stack_pointer(-16),a=X(e,V.__wbindgen_export_1,V.__wbindgen_export_2),o=Y;V.get_encoding(i,a,o,q(t));var n=Q().getInt32(i+0,!0),r=Q().getInt32(i+4,!0);if(Q().getInt32(i+8,!0))throw J(r);return Sr.__wrap(n)}finally{V.__wbindgen_add_to_stack_pointer(16)}}function yr(e,t){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);try{let i=V.__wbindgen_add_to_stack_pointer(-16),a=X(e,V.__wbindgen_export_1,V.__wbindgen_export_2),o=Y;V.encoding_for_model(i,a,o,q(t));var n=Q().getInt32(i+0,!0),r=Q().getInt32(i+4,!0);if(Q().getInt32(i+8,!0))throw J(r);return Sr.__wrap(n)}finally{V.__wbindgen_add_to_stack_pointer(16)}}function br(e){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);let t,n;try{let l=V.__wbindgen_add_to_stack_pointer(-16),u=X(e,V.__wbindgen_export_1,V.__wbindgen_export_2),d=Y;V.get_encoding_name_for_model(l,u,d);var r=Q().getInt32(l+0,!0),i=Q().getInt32(l+4,!0),a=Q().getInt32(l+8,!0),o=Q().getInt32(l+12,!0),s=r,c=i;if(o)throw s=0,c=0,J(a);return t=s,n=c,W(s,c)}finally{V.__wbindgen_add_to_stack_pointer(16),V.__wbindgen_export_3(t,n,1)}}const xr=typeof FinalizationRegistry>`u`?{register:()=>{},unregister:()=>{}}:new FinalizationRegistry(e=>V.__wbg_tiktoken_free(e>>>0,1));var Sr=class e{static __wrap(t){t>>>=0;let n=Object.create(e.prototype);return n.__wbg_ptr=t,xr.register(n,n.__wbg_ptr,n),n}__destroy_into_raw(){let e=this.__wbg_ptr;return this.__wbg_ptr=0,xr.unregister(this),e}free(){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);let e=this.__destroy_into_raw();V.__wbg_tiktoken_free(e,0)}constructor(e,t,n){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);let r=X(e,V.__wbindgen_export_1,V.__wbindgen_export_2),i=Y,a=X(n,V.__wbindgen_export_1,V.__wbindgen_export_2),o=Y,s=V.tiktoken_new(r,i,q(t),a,o);return this.__wbg_ptr=s>>>0,xr.register(this,this.__wbg_ptr,this),this}get name(){try{let n=V.__wbindgen_add_to_stack_pointer(-16);V.tiktoken_name(n,this.__wbg_ptr);var e=Q().getInt32(n+0,!0),t=Q().getInt32(n+4,!0);let r;return e!==0&&(r=W(e,t).slice(),V.__wbindgen_export_3(e,t*1,1)),r}finally{V.__wbindgen_add_to_stack_pointer(16)}}encode(e,t,n){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);try{let s=V.__wbindgen_add_to_stack_pointer(-16),c=X(e,V.__wbindgen_export_1,V.__wbindgen_export_2),l=Y;V.tiktoken_encode(s,this.__wbg_ptr,c,l,q(t),q(n));var r=Q().getInt32(s+0,!0),i=Q().getInt32(s+4,!0),a=Q().getInt32(s+8,!0);if(Q().getInt32(s+12,!0))throw J(a);var o=mr(r,i).slice();return V.__wbindgen_export_3(r,i*4,4),o}finally{V.__wbindgen_add_to_stack_pointer(16)}}encode_ordinary(e){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);try{let i=V.__wbindgen_add_to_stack_pointer(-16),a=X(e,V.__wbindgen_export_1,V.__wbindgen_export_2),o=Y;V.tiktoken_encode_ordinary(i,this.__wbg_ptr,a,o);var t=Q().getInt32(i+0,!0),n=Q().getInt32(i+4,!0),r=mr(t,n).slice();return V.__wbindgen_export_3(t,n*4,4),r}finally{V.__wbindgen_add_to_stack_pointer(16)}}encode_with_unstable(e,t,n){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);try{let a=V.__wbindgen_add_to_stack_pointer(-16),o=X(e,V.__wbindgen_export_1,V.__wbindgen_export_2),s=Y;V.tiktoken_encode_with_unstable(a,this.__wbg_ptr,o,s,q(t),q(n));var r=Q().getInt32(a+0,!0),i=Q().getInt32(a+4,!0);if(Q().getInt32(a+8,!0))throw J(i);return J(r)}finally{V.__wbindgen_add_to_stack_pointer(16)}}encode_single_token(e){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);let t=hr(e,V.__wbindgen_export_1),n=Y;return V.tiktoken_encode_single_token(this.__wbg_ptr,t,n)>>>0}decode(e){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);try{let i=V.__wbindgen_add_to_stack_pointer(-16),a=gr(e,V.__wbindgen_export_1),o=Y;V.tiktoken_decode(i,this.__wbg_ptr,a,o);var t=Q().getInt32(i+0,!0),n=Q().getInt32(i+4,!0),r=_r(t,n).slice();return V.__wbindgen_export_3(t,n*1,1),r}finally{V.__wbindgen_add_to_stack_pointer(16)}}decode_single_token_bytes(e){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);try{let i=V.__wbindgen_add_to_stack_pointer(-16);V.tiktoken_decode_single_token_bytes(i,this.__wbg_ptr,e);var t=Q().getInt32(i+0,!0),n=Q().getInt32(i+4,!0),r=_r(t,n).slice();return V.__wbindgen_export_3(t,n*1,1),r}finally{V.__wbindgen_add_to_stack_pointer(16)}}token_byte_values(){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);return J(V.tiktoken_token_byte_values(this.__wbg_ptr))}};function Cr(){return or(function(e,t){return q(JSON.parse(W(e,t)))},arguments)}function wr(){return or(function(e){return q(JSON.stringify(sr(e)))},arguments)}function Tr(e,t){return q(Error(W(e,t)))}function Er(e){return sr(e)===void 0}function Dr(e){J(e)}function Or(e,t){if(V==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);let n=sr(t),r=typeof n==`string`?n:void 0;var i=dr(r)?0:X(r,V.__wbindgen_export_1,V.__wbindgen_export_2),a=Y;Q().setInt32(e+4,a,!0),Q().setInt32(e+0,i,!0)}function kr(e,t){throw Error(W(e,t))}let Ar=!1;async function jr(e){if(Ar)return rr;let t=await e({"./tiktoken_bg.js":rr}),n=`instance`in t&&t.instance instanceof WebAssembly.Instance?t.instance:t instanceof WebAssembly.Instance?t:null;if(n==null)throw Error(`Missing instance`);return ir(n.exports),Ar=!0,rr}const Mr=new URL(`../tiktoken_bg.wasm`,import.meta.url).href;await(async()=>{await jr(async e=>{let t=e,n=await fetch(Mr);if(!n.ok)throw Error(`Failed to load tiktoken WASM from ${JSON.stringify(Mr)}: ${n.status} ${n.statusText}`);if(typeof WebAssembly.instantiateStreaming==`function`)try{return await WebAssembly.instantiateStreaming(n.clone(),t)}catch{}return WebAssembly.instantiate(await n.arrayBuffer(),t)})})();var Nr=class extends pt{disposeState(e){e.free()}tokenizeWithState(e,t){let n=this.encodeWithState(e,t);return dt(n,n.map(e=>t.decode_single_token_bytes(e).length))}},Pr=class extends Nr{modelId;constructor(e){super(),this.modelId=e}createState(){return vr(Qn(j(this.modelId,`config.msgpack`),`encoding`))}encodeWithState(e,t){return[...t.encode(e)]}},Fr=class extends Nr{modelId;constructor(e){super(),this.modelId=e}createState(){let e=j(this.modelId,`config.msgpack`);return new Sr(Ye(this.modelId,`tiktoken.model`),B(Qn(e,`special_tokens`)),Qn(e,`pat_str`))}encodeWithState(e,t){return[...t.encode(e)]}};const $=new Map,Ir=e=>{let t=$.get(e);if(t)return t;let n=ke[e],r;switch(n.kind){case`tiktoken-builtin`:r=new Pr(e);break;case`tiktoken-custom`:r=new Fr(e);break;case`huggingface`:r=new nr(e);break;case`clip-bpe`:r=new wt(e);break}return $.set(e,r),r},Lr=e=>{if(e){$.get(e)?.free(),$.delete(e);return}for(let e of $.values())e.free();$.clear()},Rr=(e,t)=>{if(typeof t==`string`){let[e]=it(t);return e}let n=t?.model;if(typeof n!=`string`)throw TypeError(`${e}() requires a single model ID as the second argument.`);let[r]=it(n);return r};function zr(e,t){return Ir(Rr(`tokenizeLoaded`,t)).tokenize(e)}function Br(e,t){return Ir(Rr(`countLoaded`,t)).getTokenCount(e)}async function Vr(e){let t=it(e);return await Ke(t),typeof e==`string`?t[0]:t}async function Hr(e,t){let n=Rr(`tokenize`,t);return await Vr(n),zr(e,n)}async function Ur(e,t){let n=Rr(`count`,t);return await Vr(n),Br(e,n)}const Wr=e=>{Je(e),Lr(e)};export{Hr as a,ke as c,Vr as i,Br as n,zr as o,Wr as r,E as s,Ur as t};
+import{n as e,r as t}from"./chunk.js";var n;try{n=new TextDecoder}catch{}var r,i,a=0;const o=[];var s=o,c=0,l={},u,d,f=0,p=0,m,h,g=[],_,ee={useRecords:!1,mapsAsObjects:!0};const te=new class{};te.name=`MessagePack 0xC1`;var v=!1,ne=2,y=class e{constructor(e){e&&(e.useRecords===!1&&e.mapsAsObjects===void 0&&(e.mapsAsObjects=!0),e.sequential&&e.trusted!==!1&&(e.trusted=!0,!e.structures&&e.useRecords!=0&&(e.structures=[],e.maxSharedStructures||=0)),e.structures?e.structures.sharedLength=e.structures.length:e.getStructures&&((e.structures=[]).uninitialized=!0,e.structures.sharedLength=0),e.int64AsNumber&&(e.int64AsType=`number`)),Object.assign(this,e)}unpack(t,n){if(r)return Ee(()=>(De(),this?this.unpack(t,n):e.prototype.unpack.call(ee,t,n)));!t.buffer&&t.constructor===ArrayBuffer&&(t=typeof Buffer<`u`?Buffer.from(t):new Uint8Array(t)),typeof n==`object`?(i=n.end||t.length,a=n.start||0):(a=0,i=n>-1?n:t.length),c=0,p=0,d=null,s=o,m=null,r=t;try{_=t.dataView||=new DataView(t.buffer,t.byteOffset,t.byteLength)}catch(e){throw r=null,t instanceof Uint8Array?e:Error(`Source must be a Uint8Array or Buffer but was a `+(t&&typeof t==`object`?t.constructor.name:typeof t))}if(this instanceof e){if(l=this,this.structures)return u=this.structures,b(n);(!u||u.length>0)&&(u=[])}else l=ee,(!u||u.length>0)&&(u=[]);return b(n)}unpackMultiple(e,t){let n,r=0;try{v=!0;let i=e.length,o=this?this.unpack(e,i):T.unpack(e,i);if(t){if(t(o,r,a)===!1)return;for(;a<i;)if(r=a,t(b(),r,a)===!1)return}else{for(n=[o];a<i;)r=a,n.push(b());return n}}catch(e){throw e.lastPosition=r,e.values=n,e}finally{v=!1,De()}}_mergeStructures(e,t){this._onLoadedStructures&&(e=this._onLoadedStructures(e)),e||=[],Object.isFrozen(e)&&(e=e.map(e=>e.slice(0)));for(let t=0,n=e.length;t<n;t++){let n=e[t];n&&(n.isShared=!0,t>=32&&(n.highByte=t-32>>5))}e.sharedLength=e.length;for(let n in t||[])if(n>=0){let r=e[n],i=t[n];i&&(r&&((e.restoreStructures||=[])[n]=r),e[n]=i)}return this.structures=e}decode(e,t){return this.unpack(e,t)}};function b(e){try{if(!l.trusted&&!v){let e=u.sharedLength||0;e<u.length&&(u.length=e)}let t;if(l._readStruct&&r[a]<64&&r[a]>=32?(t=l._readStruct(r,a,i),r=null,!(e&&e.lazy)&&t&&(t=t.toJSON()),a=i):t=x(),m&&=(a=m.postBundlePosition,null),v&&(u.restoreStructures=null),a==i)u&&u.restoreStructures&&re(),u=null,r=null,h&&=null;else if(a>i)throw Error(`Unexpected end of MessagePack data`);else if(!v){let e;try{e=JSON.stringify(t,(e,t)=>typeof t==`bigint`?`${t}n`:t).slice(0,100)}catch(t){e=`(JSON view not available `+t+`)`}throw Error(`Data read, but end of buffer not reached `+e)}return t}catch(e){throw u&&u.restoreStructures&&re(),De(),(e instanceof RangeError||e.message.startsWith(`Unexpected end of buffer`)||a>i)&&(e.incomplete=!0),e}}function re(){for(let e in u.restoreStructures)u[e]=u.restoreStructures[e];u.restoreStructures=null}function x(){let e=r[a++];if(e<160)if(e<128){if(e<64)return e;{let t=u[e&63]||l.getStructures&&se()[e&63];return t?(t.read||=ae(t,e&63),t.read()):e}}else if(e<144)if(e-=128,l.mapsAsObjects){let t={};for(let n=0;n<e;n++){let e=ye();e===`__proto__`&&(e=`__proto_`),t[e]=x()}return t}else{let t=new Map;for(let n=0;n<e;n++)t.set(x(),x());return t}else{e-=144;let t=Array(e);for(let n=0;n<e;n++)t[n]=x();return l.freezeData?Object.freeze(t):t}else if(e<192){let t=e-160;if(p>=a)return d.slice(a-f,(a+=t)-f);if(p==0&&i<140){let e=t<16?he(t):me(t);if(e!=null)return e}return ce(t)}else{let t;switch(e){case 192:return null;case 193:return m?(t=x(),t>0?m[1].slice(m.position1,m.position1+=t):m[0].slice(m.position0,m.position0-=t)):te;case 194:return!1;case 195:return!0;case 196:if(t=r[a++],t===void 0)throw Error(`Unexpected end of buffer`);return _e(t);case 197:return t=_.getUint16(a),a+=2,_e(t);case 198:return t=_.getUint32(a),a+=4,_e(t);case 199:return w(r[a++]);case 200:return t=_.getUint16(a),a+=2,w(t);case 201:return t=_.getUint32(a),a+=4,w(t);case 202:if(t=_.getFloat32(a),l.useFloat32>2){let e=Oe[(r[a]&127)<<1|r[a+1]>>7];return a+=4,(e*t+(t>0?.5:-.5)>>0)/e}return a+=4,t;case 203:return t=_.getFloat64(a),a+=8,t;case 204:return r[a++];case 205:return t=_.getUint16(a),a+=2,t;case 206:return t=_.getUint32(a),a+=4,t;case 207:return l.int64AsType===`number`?(t=_.getUint32(a)*4294967296,t+=_.getUint32(a+4)):l.int64AsType===`string`?t=_.getBigUint64(a).toString():l.int64AsType===`auto`?(t=_.getBigUint64(a),t<=BigInt(2)<<BigInt(52)&&(t=Number(t))):t=_.getBigUint64(a),a+=8,t;case 208:return _.getInt8(a++);case 209:return t=_.getInt16(a),a+=2,t;case 210:return t=_.getInt32(a),a+=4,t;case 211:return l.int64AsType===`number`?(t=_.getInt32(a)*4294967296,t+=_.getUint32(a+4)):l.int64AsType===`string`?t=_.getBigInt64(a).toString():l.int64AsType===`auto`?(t=_.getBigInt64(a),t>=BigInt(-2)<<BigInt(52)&&t<=BigInt(2)<<BigInt(52)&&(t=Number(t))):t=_.getBigInt64(a),a+=8,t;case 212:if(t=r[a++],t==114)return xe(r[a++]&63);{let e=g[t];if(e)return e.read?(a++,e.read(x())):e.noBuffer?(a++,e()):e(r.subarray(a,++a));throw Error(`Unknown extension `+t)}case 213:return t=r[a],t==114?(a++,xe(r[a++]&63,r[a++])):w(2);case 214:return w(4);case 215:return w(8);case 216:return w(16);case 217:return t=r[a++],p>=a?d.slice(a-f,(a+=t)-f):le(t);case 218:return t=_.getUint16(a),a+=2,p>=a?d.slice(a-f,(a+=t)-f):ue(t);case 219:return t=_.getUint32(a),a+=4,p>=a?d.slice(a-f,(a+=t)-f):de(t);case 220:return t=_.getUint16(a),a+=2,fe(t);case 221:return t=_.getUint32(a),a+=4,fe(t);case 222:return t=_.getUint16(a),a+=2,pe(t);case 223:return t=_.getUint32(a),a+=4,pe(t);default:if(e>=224)return e-256;if(e===void 0){let e=Error(`Unexpected end of MessagePack data`);throw e.incomplete=!0,e}throw Error(`Unknown MessagePack token `+e)}}}const ie=/^[a-zA-Z_$][a-zA-Z\d_$]*$/;function ae(e,t){function n(){if(n.count++>ne){let r;try{r=e.read=Function(`r`,`return function(){return `+(l.freezeData?`Object.freeze`:``)+`({`+e.map(e=>e===`__proto__`?`__proto_:r()`:ie.test(e)?e+`:r()`:`[`+JSON.stringify(e)+`]:r()`).join(`,`)+`})}`)(x)}catch{return ne=1/0,n()}return e.highByte===0&&(e.read=oe(t,e.read)),r()}let r={};for(let t=0,n=e.length;t<n;t++){let n=e[t];n===`__proto__`&&(n=`__proto_`),r[n]=x()}return l.freezeData?Object.freeze(r):r}return n.count=0,e.highByte===0?oe(t,n):n}const oe=(e,t)=>function(){let n=r[a++];if(n===0)return t();let i=e<32?-(e+(n<<5)):e+(n<<5),o=u[i]||se()[i];if(!o)throw Error(`Record id is not defined for `+i);return o.read||=ae(o,e),o.read()};function se(){let e=Ee(()=>(r=null,l.getStructures()));return u=l._mergeStructures(e,u)}var ce=S,le=S,ue=S,de=S;function S(e){let t;if(e<16&&(t=he(e)))return t;if(e>64&&n)return n.decode(r.subarray(a,a+=e));let i=a+e,o=[];for(t=``;a<i;){let e=r[a++];if(!(e&128))o.push(e);else if((e&224)==192){let t=r[a++]&63,n=(e&31)<<6|t;n<128?o.push(65533):o.push(n)}else if((e&240)==224){let t=r[a++]&63,n=r[a++]&63,i=(e&31)<<12|t<<6|n;i<2048||i>=55296&&i<=57343?o.push(65533):o.push(i)}else if((e&248)==240){let t=r[a++]&63,n=r[a++]&63,i=r[a++]&63,s=(e&7)<<18|t<<12|n<<6|i;s<65536||s>1114111?o.push(65533):s>65535?(s-=65536,o.push(s>>>10&1023|55296),s=56320|s&1023,o.push(s)):o.push(s)}else o.push(65533);o.length>=4096&&(t+=C.apply(String,o),o.length=0)}return o.length>0&&(t+=C.apply(String,o)),t}function fe(e){let t=Array(e);for(let n=0;n<e;n++)t[n]=x();return l.freezeData?Object.freeze(t):t}function pe(e){if(l.mapsAsObjects){let t={};for(let n=0;n<e;n++){let e=ye();e===`__proto__`&&(e=`__proto_`),t[e]=x()}return t}else{let t=new Map;for(let n=0;n<e;n++)t.set(x(),x());return t}}var C=String.fromCharCode;function me(e){let t=a,n=Array(e);for(let i=0;i<e;i++){let e=r[a++];if((e&128)>0){a=t;return}n[i]=e}return C.apply(String,n)}function he(e){if(e<4)if(e<2){if(e===0)return``;{let e=r[a++];if((e&128)>1){--a;return}return C(e)}}else{let t=r[a++],n=r[a++];if((t&128)>0||(n&128)>0){a-=2;return}if(e<3)return C(t,n);let i=r[a++];if((i&128)>0){a-=3;return}return C(t,n,i)}else{let t=r[a++],n=r[a++],i=r[a++],o=r[a++];if((t&128)>0||(n&128)>0||(i&128)>0||(o&128)>0){a-=4;return}if(e<6){if(e===4)return C(t,n,i,o);{let e=r[a++];if((e&128)>0){a-=5;return}return C(t,n,i,o,e)}}else if(e<8){let s=r[a++],c=r[a++];if((s&128)>0||(c&128)>0){a-=6;return}if(e<7)return C(t,n,i,o,s,c);let l=r[a++];if((l&128)>0){a-=7;return}return C(t,n,i,o,s,c,l)}else{let s=r[a++],c=r[a++],l=r[a++],u=r[a++];if((s&128)>0||(c&128)>0||(l&128)>0||(u&128)>0){a-=8;return}if(e<10){if(e===8)return C(t,n,i,o,s,c,l,u);{let e=r[a++];if((e&128)>0){a-=9;return}return C(t,n,i,o,s,c,l,u,e)}}else if(e<12){let d=r[a++],f=r[a++];if((d&128)>0||(f&128)>0){a-=10;return}if(e<11)return C(t,n,i,o,s,c,l,u,d,f);let p=r[a++];if((p&128)>0){a-=11;return}return C(t,n,i,o,s,c,l,u,d,f,p)}else{let d=r[a++],f=r[a++],p=r[a++],m=r[a++];if((d&128)>0||(f&128)>0||(p&128)>0||(m&128)>0){a-=12;return}if(e<14){if(e===12)return C(t,n,i,o,s,c,l,u,d,f,p,m);{let e=r[a++];if((e&128)>0){a-=13;return}return C(t,n,i,o,s,c,l,u,d,f,p,m,e)}}else{let h=r[a++],g=r[a++];if((h&128)>0||(g&128)>0){a-=14;return}if(e<15)return C(t,n,i,o,s,c,l,u,d,f,p,m,h,g);let _=r[a++];if((_&128)>0){a-=15;return}return C(t,n,i,o,s,c,l,u,d,f,p,m,h,g,_)}}}}}function ge(){let e=r[a++],t;if(e<192)t=e-160;else switch(e){case 217:t=r[a++];break;case 218:t=_.getUint16(a),a+=2;break;case 219:t=_.getUint32(a),a+=4;break;default:throw Error(`Expected string`)}return S(t)}function _e(e){return l.copyBuffers?Uint8Array.prototype.slice.call(r,a,a+=e):r.subarray(a,a+=e)}function w(e){let t=r[a++];if(g[t]){let n;return g[t](r.subarray(a,n=a+=e),e=>{a=e;try{return x()}finally{a=n}})}else throw Error(`Unknown extension type `+t)}var ve=Array(4096);function ye(){let e=r[a++];if(e>=160&&e<192){if(e-=160,p>=a)return d.slice(a-f,(a+=e)-f);if(!(p==0&&i<180))return ce(e)}else return a--,be(x());let t=(e<<5^(e>1?_.getUint16(a):e>0?r[a]:0))&4095,n=ve[t],o=a,s=a+e-3,c,l=0;if(n&&n.bytes==e){for(;o<s;){if(c=_.getUint32(o),c!=n[l++]){o=1879048192;break}o+=4}for(s+=3;o<s;)if(c=r[o++],c!=n[l++]){o=1879048192;break}if(o===s)return a=o,n.string;s-=3,o=a}for(n=[],ve[t]=n,n.bytes=e;o<s;)c=_.getUint32(o),n.push(c),o+=4;for(s+=3;o<s;)c=r[o++],n.push(c);let u=e<16?he(e):me(e);return u==null?n.string=ce(e):n.string=u}function be(e){if(typeof e==`string`)return e;if(typeof e==`number`||typeof e==`boolean`||typeof e==`bigint`)return e.toString();if(e==null)return e+``;if(l.allowArraysInMapKeys&&Array.isArray(e)&&e.flat().every(e=>[`string`,`number`,`boolean`,`bigint`].includes(typeof e)))return e.flat().toString();throw Error(`Invalid property type for record: ${typeof e}`)}const xe=(e,t)=>{let n=x().map(be),r=e;t!==void 0&&(e=e<32?-((t<<5)+e):(t<<5)+e,n.highByte=t);let i=u[e];return i&&(i.isShared||v)&&((u.restoreStructures||=[])[e]=i),u[e]=n,n.read=ae(n,r),n.read()};g[0]=()=>{},g[0].noBuffer=!0,g[66]=e=>{let t=e.byteLength%8||8,n=BigInt(e[0]&128?e[0]-256:e[0]);for(let r=1;r<t;r++)n<<=BigInt(8),n+=BigInt(e[r]);if(e.byteLength!==t){let r=new DataView(e.buffer,e.byteOffset,e.byteLength),i=(e,t)=>{let n=t-e;if(n<=40){let n=r.getBigUint64(e);for(let i=e+8;i<t;i+=8)n<<=BigInt(64),n|=r.getBigUint64(i);return n}let a=e+(n>>4<<3),o=i(e,a),s=i(a,t);return o<<BigInt((t-a)*8)|s};n=n<<BigInt((r.byteLength-t)*8)|i(t,r.byteLength)}return n};let Se={Error,EvalError,RangeError,ReferenceError,SyntaxError,TypeError,URIError,AggregateError:typeof AggregateError==`function`?AggregateError:null};g[101]=()=>{let e=x();if(!Se[e[0]]){let t=Error(e[1],{cause:e[2]});return t.name=e[0],t}return Se[e[0]](e[1],{cause:e[2]})},g[105]=e=>{if(l.structuredClone===!1)throw Error(`Structured clone extension is disabled`);let t=_.getUint32(a-4);h||=new Map;let n=r[a],i;i=n>=144&&n<160||n==220||n==221?[]:n>=128&&n<144||n==222||n==223?new Map:(n>=199&&n<=201||n>=212&&n<=216)&&r[a+1]===115?new Set:{};let o={target:i};h.set(t,o);let s=x();if(o.used)Object.assign(i,s);else return o.target=s;if(i instanceof Map)for(let[e,t]of s.entries())i.set(e,t);if(i instanceof Set)for(let e of Array.from(s))i.add(e);return i},g[112]=e=>{if(l.structuredClone===!1)throw Error(`Structured clone extension is disabled`);let t=_.getUint32(a-4),n=h.get(t);return n.used=!0,n.target},g[115]=()=>new Set(x());const Ce=[`Int8`,`Uint8`,`Uint8Clamped`,`Int16`,`Uint16`,`Int32`,`Uint32`,`Float32`,`Float64`,`BigInt64`,`BigUint64`].map(e=>e+`Array`);let we=typeof globalThis==`object`?globalThis:window;g[116]=e=>{let t=e[0],n=Uint8Array.prototype.slice.call(e,1).buffer,r=Ce[t];if(!r){if(t===16)return n;if(t===17)return new DataView(n);throw Error(`Could not find typed array for code `+t)}return new we[r](n)},g[120]=()=>{let e=x();return new RegExp(e[0],e[1])};const Te=[];g[98]=e=>{let t=(e[0]<<24)+(e[1]<<16)+(e[2]<<8)+e[3],n=a;return a+=t-e.length,m=Te,m=[ge(),ge()],m.position0=0,m.position1=0,m.postBundlePosition=a,a=n,x()},g[255]=e=>e.length==4?new Date((e[0]*16777216+(e[1]<<16)+(e[2]<<8)+e[3])*1e3):e.length==8?new Date(((e[0]<<22)+(e[1]<<14)+(e[2]<<6)+(e[3]>>2))/1e6+((e[3]&3)*4294967296+e[4]*16777216+(e[5]<<16)+(e[6]<<8)+e[7])*1e3):e.length==12?new Date(((e[0]<<24)+(e[1]<<16)+(e[2]<<8)+e[3])/1e6+((e[4]&128?-281474976710656:0)+e[6]*1099511627776+e[7]*4294967296+e[8]*16777216+(e[9]<<16)+(e[10]<<8)+e[11])*1e3):new Date(`invalid`);function Ee(e){l&&l._onSaveState&&l._onSaveState();let t=i,n=a,o=c,g=f,ee=p,te=d,ne=s,y=h,b=m,re=new Uint8Array(r.slice(0,i)),x=u,ie=u.slice(0,u.length),ae=l,oe=v,se=e();return i=t,a=n,c=o,f=g,p=ee,d=te,s=ne,h=y,m=b,r=re,v=oe,u=x,u.splice(0,u.length,...ie),l=ae,_=new DataView(r.buffer,r.byteOffset,r.byteLength),se}function De(){r=null,h=null,u=null}const Oe=Array(147);for(let e=0;e<256;e++)Oe[e]=+(`1e`+Math.floor(45.15-e*.30103));var T=new y({useRecords:!1});T.unpack,T.unpackMultiple,T.unpack,new Uint8Array(new Float32Array(1).buffer,0,4),y.SUPPORTS_STRUCT_HOOKS=!0;const ke={gpt:{encoding:`o200k_base`,kind:`tiktoken-builtin`,openrouter:`openai/gpt-5.5`,source:{encodingJsonUrl:`https://tiktoken.pages.dev/js/o200k_base.json`},title:`GPT-5.5`},gemma:{kind:`huggingface`,openrouter:`google/gemma-4-31b-it`,source:{tokenizerConfigUrl:`https://huggingface.co/google/gemma-4-31B-it/resolve/main/tokenizer_config.json`,tokenizerJsonUrl:`https://huggingface.co/google/gemma-4-31B-it/resolve/main/tokenizer.json`},title:`Gemma 4 31B it`},qwen:{kind:`huggingface`,openrouter:`qwen/qwen3.6-27b`,source:{tokenizerConfigUrl:`https://huggingface.co/Qwen/Qwen3.6-27B/resolve/main/tokenizer_config.json`,tokenizerJsonUrl:`https://huggingface.co/Qwen/Qwen3.6-27B/resolve/main/tokenizer.json`},title:`Qwen 3.6 27B`},kimi:{kind:`tiktoken-custom`,openrouter:`moonshotai/kimi-k2.6`,source:{modelUrl:`https://huggingface.co/moonshotai/Kimi-K2.6/resolve/main/tiktoken.model`,tokenizerConfigUrl:`https://huggingface.co/moonshotai/Kimi-K2.6/resolve/main/tokenizer_config.json`,tokenizerImplementationUrl:`https://huggingface.co/moonshotai/Kimi-K2.6/resolve/main/tokenization_kimi.py`},title:`Kimi K2.6`},deepseek:{kind:`huggingface`,openrouter:`deepseek/deepseek-v4-pro`,source:{tokenizerConfigUrl:`https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/resolve/main/tokenizer_config.json`,tokenizerJsonUrl:`https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/resolve/main/tokenizer.json`},title:`DeepSeek V4 Pro`},mimo:{kind:`huggingface`,openrouter:`xiaomi/mimo-v2.5-pro`,source:{tokenizerConfigUrl:`https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro/resolve/main/tokenizer_config.json`,tokenizerJsonUrl:`https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro/resolve/main/tokenizer.json`},title:`MiMo V2.5 Pro`},sdxl:{kind:`clip-bpe`,source:{mergesUrl:`https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/resolve/main/tokenizer_2/merges.txt`,specialTokensMapUrl:`https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/resolve/main/tokenizer_2/special_tokens_map.json`,tokenizerConfigUrl:`https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/resolve/main/tokenizer_2/tokenizer_config.json`,vocabUrl:`https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/resolve/main/tokenizer_2/vocab.json`},title:`Stable Diffusion XL`},glm:{kind:`huggingface`,openrouter:`zai-org/glm-5.1`,source:{tokenizerConfigUrl:`https://huggingface.co/zai-org/GLM-5.1/resolve/main/tokenizer_config.json`,tokenizerJsonUrl:`https://huggingface.co/zai-org/GLM-5.1/resolve/main/tokenizer.json`},title:`GLM 5.1`},minimax:{kind:`huggingface`,openrouter:`minimax/minimax-m2.7`,source:{tokenizerConfigUrl:`https://huggingface.co/MiniMaxAI/MiniMax-M2.7/resolve/main/tokenizer_config.json`,tokenizerJsonUrl:`https://huggingface.co/MiniMaxAI/MiniMax-M2.7/resolve/main/tokenizer.json`},title:`MiniMax M2.7`},hy:{kind:`huggingface`,openrouter:`tencent/hy3-preview`,title:`Hy3 Preview`,source:{tokenizerConfigUrl:`https://huggingface.co/tencent/Hy3-preview/resolve/main/tokenizer_config.json`,tokenizerJsonUrl:`https://huggingface.co/tencent/Hy3-preview/resolve/main/tokenizer.json`}},step:{kind:`huggingface`,openrouter:`stepfun/step-3.7-flash`,source:{specialTokensMapUrl:`https://huggingface.co/stepfun-ai/Step-3.7-Flash/resolve/main/special_tokens_map.json`,tokenizerConfigUrl:`https://huggingface.co/stepfun-ai/Step-3.7-Flash/resolve/main/tokenizer_config.json`,tokenizerJsonUrl:`https://huggingface.co/stepfun-ai/Step-3.7-Flash/resolve/main/tokenizer.json`},title:`Step 3.7 Flash`}},E=Object.keys(ke),Ae=new y({mapsAsObjects:!1}),je=new TextDecoder,Me=Object.create(null),D=new Map,Ne=Object.create(null),Pe=new Map,Fe=new Map;let Ie;const Le=(e,t)=>`${e}/${t}`,Re=e=>Ne[e]??0,ze=e=>{let t=Re(e)+1;return Ne[e]=t,t},Be=e=>{let t=`${e}/`;for(let e of Fe.keys())e.startsWith(t)&&Fe.delete(e);for(let e of Pe.keys())e.startsWith(t)&&Pe.delete(e)},Ve=e=>{let t=Me[e];if(!t)throw Error(`Missing tokenizer assets for model ${JSON.stringify(e)}. Call load() first or use tokenize()/count() to auto-load it.`);return t},He=()=>{if(!Ie)throw Error(`No tokenizer asset loader is registered for the current entry.`);return Ie},Ue=(e,t)=>{let n=Ve(e);if(!Object.hasOwn(n,t))throw Error(`Missing tokenizer asset ${JSON.stringify(t)} for model ${JSON.stringify(e)}. Run “bun run fetch” first.`);return n[t]},We=(e,t)=>Ue(e,t),Ge=e=>{if(e.endsWith(`.msgpack`))return e;if(e.endsWith(`.json`))return`${e.slice(0,-5)}.msgpack`;throw TypeError(`Expected a MessagePack or JSON file name, got ${JSON.stringify(e)}.`)},Ke=e=>Object.hasOwn(Me,e),qe=e=>{Ie=e},Je=(e,t)=>{Me[e]=t,Be(e)},Ye=async e=>{await Promise.all(e.map(async e=>{if(Ke(e))return;let t=D.get(e);if(t){await t;return}let n=Re(e),r=(async()=>{let t=await He().load(e);Re(e)===n&&Je(e,t)})().finally(()=>{D.get(e)===r&&D.delete(e)});D.set(e,r),await r}))},Xe=e=>{ze(e),delete Me[e],D.delete(e),Be(e)},Ze=e=>{if(e){Xe(e);return}for(let e of E)Xe(e)},Qe=(e,t)=>{if(t.endsWith(`.json`)||t.endsWith(`.msgpack`))throw TypeError(`Cannot read structured tokenizer asset ${JSON.stringify(t)} as text.`);let n=Le(e,t),r=Fe.get(n);if(r!==void 0)return r;let i=je.decode(We(e,t));return Fe.set(n,i),i},O=(e,t)=>{let n=Ge(t),r=Le(e,n),i=Pe.get(r);if(i!==void 0)return i;let a=Ae.unpack(We(e,n));return Pe.set(r,a),a},$e={gpt:new URL(`../gpt.bin`,import.meta.url),gemma:new URL(`../gemma.bin`,import.meta.url),qwen:new URL(`../qwen.bin`,import.meta.url),kimi:new URL(`../kimi.bin`,import.meta.url),deepseek:new URL(`../deepseek.bin`,import.meta.url),mimo:new URL(`../mimo.bin`,import.meta.url),sdxl:new URL(`../sdxl.bin`,import.meta.url),glm:new URL(`../glm.bin`,import.meta.url),minimax:new URL(`../minimax.bin`,import.meta.url),hy:new URL(`../hy.bin`,import.meta.url),step:new URL(`../step.bin`,import.meta.url)};let et;const tt=()=>(et??=(async()=>(await import(`./decompress.js`).then(e=>t(e.default,1))).default)(),et),nt=async e=>{if(typeof DecompressionStream==`function`)try{let t=new Blob([e]).stream().pipeThrough(new DecompressionStream(`brotli`));return new Uint8Array(await new Response(t).arrayBuffer())}catch{}return(await tt())(e)},rt=new y({mapsAsObjects:!1}),it=e=>{if(e instanceof Uint8Array)return e;if(e instanceof ArrayBuffer)return new Uint8Array(e);if(ArrayBuffer.isView(e))return new Uint8Array(e.buffer,e.byteOffset,e.byteLength);throw TypeError(`Expected bundled tokenizer asset contents to be binary, got ${typeof e}.`)},at=e=>{if(!(e instanceof Map))throw TypeError(`Expected a tokenizer asset bundle to unpack into a Map.`);let t=Object.create(null);for(let[n,r]of e.entries()){if(typeof n!=`string`)throw TypeError(`Expected tokenizer asset file names to be strings, got ${typeof n}.`);t[n]=it(r)}return t};var ot=class{async load(e){let t=await nt(await this.readCompressedBundle(e));return at(rt.unpack(t))}};qe(new class extends ot{bundleUrls;constructor(e){super(),this.bundleUrls=e}async readCompressedBundle(e){let t=this.bundleUrls[e],n=await fetch(t);if(!n.ok)throw Error(`Failed to fetch tokenizer assets for model ${JSON.stringify(e)} from ${JSON.stringify(t.href)}: ${n.status} ${n.statusText}`);return new Uint8Array(await n.arrayBuffer())}}($e));const st=e=>{let t=[...E];e!==void 0&&(t.length=0,typeof e==`string`?t.push(e):t.push(...e));for(let e of t)if(!E.includes(e))throw Error(`Unknown model ${JSON.stringify(e)}. Supported models: ${E.join(`, `)}`);return t},ct=new TextDecoder(`utf-8`,{fatal:!0}),lt=new TextEncoder,ut=e=>{let t=[],n=0;for(let r of e)t.push(n),n+=r;return t},dt=e=>{if(typeof e==`string`)return e;try{return ct.decode(e)}catch{throw TypeError(`Tokenizer input bytes must be valid UTF-8.`)}},ft=e=>lt.encode(e).byteLength,pt=(e,t,n)=>{if(e.length!==t.length)throw TypeError(`Expected ${e.length} token start offsets, got ${t.length}.`);let r={offsets:t.slice(1),tokens:[...e]};return n===void 0?r:{...r,processedInput:n}},mt=(e,t,n)=>{if(e.length!==t.length)throw TypeError(`Expected ${e.length} token byte lengths, got ${t.length}.`);return pt(e,ut(t),n)},ht=new TextEncoder;var gt=class{#e=!1;#t;disposeState(e){}encode(e){return this.encodeWithState(dt(e),this.getState())}free(){this.#e&&=(this.disposeState(this.#t),this.#t=void 0,!1)}getState(){return this.#e||=(this.#t=this.createState(),!0),this.#t}getTokenCount(e){return this.encode(e).length}tokenize(e){let t=this.tokenizeWithState(dt(e),this.getState());return typeof e==`string`||t.processedInput===void 0?t:{...t,processedInput:ht.encode(t.processedInput)}}};const _t=/<\|startoftext\|>|<\|endoftext\|>|'d|'ll|'m|'re|'s|'t|'ve|\p{L}+|\p{N}|[^\s\p{L}\p{N}]+/gu,vt=/<\/w>$/u,yt=/\s+/gu,bt=new TextEncoder,xt=e=>{if(typeof e==`string`)return e;if(e instanceof Map){let t=e.get(`content`);if(typeof t==`string`)return t}if(e&&typeof e==`object`&&`content`in e&&typeof e.content==`string`)return e.content},St=()=>{let e=Array.from({length:256},()=>``),t=[...Array.from({length:94},(e,t)=>t+33),...Array.from({length:12},(e,t)=>t+161),...Array.from({length:82},(e,t)=>t+174)],n=[...t],r=256;for(let e=0;e<256;e++)t.includes(e)||(t.push(e),n.push(r),r++);for(let[r,i]of t.entries())e[i]=String.fromCodePoint(n[r]);return e},Ct=e=>{let t=new Map;for(let[n,r]of e.split(/\r?\n/u).filter(Boolean).filter(e=>!e.startsWith(`#`)).entries())t.set(r,n);return t},wt=e=>e.normalize(`NFC`).replaceAll(yt,` `).trim().toLowerCase(),Tt=e=>[...e.replace(vt,``)].length,Et=(e,t,n)=>n===t?e:{...e,processedInput:n};var Dt=class extends gt{modelId;#e=new Map;constructor(e){super(),this.modelId=e}createState(){let e=O(this.modelId,`vocab.msgpack`),t=O(this.modelId,`tokenizer_config.msgpack`),n=new Map,r=xt(t.get(`unk_token`))??`<|endoftext|>`,i=e.get(r);if(i===void 0)throw Error(`Could not find CLIP unknown token ${JSON.stringify(r)} in ${JSON.stringify(this.modelId)} vocabulary.`);for(let r of[t.get(`bos_token`),t.get(`eos_token`),t.get(`unk_token`)]){let t=xt(r);if(!t)continue;let i=e.get(t);i!==void 0&&n.set(t,i)}return{byteEncoder:St(),mergeRanks:Ct(Qe(this.modelId,`merges.txt`)),specialTokenIds:n,unknownTokenId:i,vocabulary:e}}encodeWithState(e,t){return this.tokenizeWithState(e,t).tokens}tokenizeWithState(e,t){let n=wt(e);if(!n)return Et({offsets:[],tokens:[]},e,n);let r=[],i=[],a=0,o=0;for(let e of n.matchAll(_t)){let s=e[0],c=e.index;a+=ft(n.slice(o,c));let l=a,u=t.specialTokenIds.get(s);if(u!==void 0){r.push(u),i.push(l),a+=ft(s),o=c+s.length;continue}let d=Array.from(bt.encode(s),e=>t.byteEncoder[e]).join(``),f=0;for(let e of this.#t(d,t.mergeRanks))r.push(t.vocabulary.get(e)??t.unknownTokenId),i.push(l+f),f+=Tt(e);a+=ft(s),o=c+s.length}return Et(pt(r,i),e,n)}#t(e,t){let n=this.#e.get(e);if(n)return n;let r=[...e];if(!r.length){let t=[];return this.#e.set(e,t),t}let i=[...r.slice(0,-1),`${r.at(-1)}</w>`];for(;i.length>1;){let e=-1,n=1/0;for(let r=0;r<i.length-1;r++){let a=t.get(`${i[r]} ${i[r+1]}`);a!==void 0&&a<n&&(n=a,e=r)}if(e<0)break;let r=[];for(let t=0;t<i.length;t++){if(t===e){r.push(`${i[t]}${i[t+1]}`),t++;continue}r.push(i[t])}i=r}return this.#e.set(e,i),i}},Ot=class{constructor(e){this.trie=this._build_trie(e)}_build_trie(e){let t=Object.create(null);for(let n of e){let e=t;for(let t=0;t<n.length;++t){let r=n[t];e=e[r]??=Object.create(null)}e.end=n}return t}split(e){let t=[],n=e.length,r=0,i=0;for(;i<n;){let a=this.trie,o=null,s=i;for(;s<n&&(a=a[e[s]]);)a.end&&(o=a.end),++s;o?(i>r&&t.push(e.slice(r,i)),t.push(o),i+=o.length,r=i):++i}return r<n&&t.push(e.slice(r)),t}},kt=class{constructor(e){this.content=e.content,this.id=e.id,this.single_word=e.single_word??!1,this.lstrip=e.lstrip??!1,this.rstrip=e.rstrip??!1,this.special=e.special??!1,this.normalized=e.normalized??!this.special}},At=(()=>{let e=[...Array.from({length:94},(e,t)=>t+33),...Array.from({length:12},(e,t)=>t+161),...Array.from({length:82},(e,t)=>t+174)],t=e.slice(),n=0;for(let r=0;r<256;++r)e.includes(r)||(e.push(r),t.push(256+n),n+=1);let r=t.map(e=>String.fromCharCode(e));return Object.fromEntries(e.map((e,t)=>[e,r[t]]))})(),jt=(e=>Object.fromEntries(Object.entries(e).map(([e,t])=>[t,e])))(At),Mt=`.,!?…。，、।۔،`,Nt=new Map([[`(?i:'s|'t|'re|'ve|'m|'ll|'d)`,`(?:'([sS]|[tT]|[rR][eE]|[vV][eE]|[mM]|[lL][lL]|[dD]))`],[`(?i:[sdmt]|ll|ve|re)`,`(?:[sS]|[dD]|[mM]|[tT]|[lL][lL]|[vV][eE]|[rR][eE])`],[`[^\\r\\n\\p{L}\\p{N}]?+`,`[^\\r\\n\\p{L}\\p{N}]?`],[`[^\\s\\p{L}\\p{N}]++`,`[^\\s\\p{L}\\p{N}]+`],[`(?>\\p{Nd}{510})`,`(?:\\p{Nd}{510})`],[`\\p{Nd}{3}+`,`(?:\\p{Nd}{3})+`],[`\\G`,``],[` ?[^(\\s|[${Mt}])]+`,` ?[^\\s${Mt}]+`]]),k=`\\p{P}\\u0021-\\u002F\\u003A-\\u0040\\u005B-\\u0060\\u007B-\\u007E`,Pt=e=>e.replace(/ \./g,`.`).replace(/ \?/g,`?`).replace(/ \!/g,`!`).replace(/ ,/g,`,`).replace(/ \' /g,`'`).replace(/ n't/g,`n't`).replace(/ 'm/g,`'m`).replace(/ 's/g,`'s`).replace(/ 've/g,`'ve`).replace(/ 're/g,`'re`),A=(e,t=!0)=>{if(e.Regex!==void 0){let t=e.Regex.replace(/\\([#&~])/g,`$1`);t=t.replace(/\\A/g,`^`).replace(/\\z/g,`$`).replace(/\\Z/g,`(?=\\r?\\n?$)`);for(let[e,n]of Nt)t=t.replaceAll(e,n);try{return new RegExp(t,`gu`)}catch(e){if(!(e instanceof SyntaxError)||!e.message.toLowerCase().includes(`invalid property name`))throw e;let n=!1,r=t.replace(/(\\[pP])\{([^}=]+)\}/g,(e,t,r)=>{try{return RegExp(`\\p{${r}}`,`u`),`${t}{${r}}`}catch{return n=!0,`${t}{Script=${r}}`}});if(!n)throw e;try{return new RegExp(r,`gu`)}catch{throw e}}}else if(e.String!==void 0){let n=Ft(e.String);return new RegExp(t?n:`(${n})`,`gu`)}else return console.warn(`Unknown pattern type:`,e),null},Ft=e=>e.replace(/[.*+?^${}()|[\]\\]/g,`\\$&`),It=(e,t,n)=>{let r=[],i=0;for(;i<e.length;){if(r.push(e[i]),(t.get(e[i])??n)!==n){++i;continue}for(;++i<e.length&&(t.get(e[i])??n)===n;)t.get(r.at(-1))!==n&&(r[r.length-1]+=e[i])}return r},Lt=e=>e>=19968&&e<=40959||e>=13312&&e<=19903||e>=131072&&e<=173791||e>=173824&&e<=177983||e>=177984&&e<=178207||e>=178208&&e<=183983||e>=63744&&e<=64255||e>=194560&&e<=195103,Rt=e=>Number.isInteger(e)||typeof e==`bigint`,zt=e=>{let t=0;for(let n of e)++t;return t},Bt=e=>Ut(e.toLowerCase()),j=(...e)=>Array.prototype.concat.apply([],e),Vt=e=>new Map(Object.entries(e)),Ht=(e,t)=>{let n=[],r=0;for(let i of e.matchAll(t)){let t=i[0];r<i.index&&n.push(e.slice(r,i.index)),t.length>0&&n.push(t),r=i.index+t.length}return r<e.length&&n.push(e.slice(r)),n},Ut=e=>e.replace(/\p{M}/gu,``),Wt=(e,t,n=[])=>{if(!e||Array.isArray(e)||typeof e!=`object`)return`${t} must be a valid object`;for(let r of n)if(!(r in e))return`${t} must contain a "${r}" property`;return null},Gt=e=>e.match(/\S+/g)||[],M=class{constructor(){let e=function(...t){return e._call(...t)};return Object.setPrototypeOf(e,new.target.prototype)}},N=class extends M{constructor(e){super(),this.config=e}_call(e){return this.normalize(e)}},Kt=class extends N{tokenize_chinese_chars(e){let t=[];for(let n=0;n<e.length;++n){let r=e[n];Lt(r.charCodeAt(0))?(t.push(` `),t.push(r),t.push(` `)):t.push(r)}return t.join(``)}strip_accents(e){return e.normalize(`NFD`).replace(/\p{Mn}/gu,``)}is_control(e){switch(e){case`	`:case`
+`:case`\r`:return!1;default:return/^\p{Cc}|\p{Cf}|\p{Co}|\p{Cs}$/u.test(e)}}clean_text(e){let t=[];for(let n of e){let e=n.charCodeAt(0);e===0||e===65533||this.is_control(n)||(/^\s$/.test(n)?t.push(` `):t.push(n))}return t.join(``)}normalize(e){return this.config.clean_text&&(e=this.clean_text(e)),this.config.handle_chinese_chars&&(e=this.tokenize_chinese_chars(e)),this.config.lowercase?(e=e.toLowerCase(),this.config.strip_accents!==!1&&(e=this.strip_accents(e))):this.config.strip_accents&&(e=this.strip_accents(e)),e}},qt=class extends N{constructor(e){super(e),this.charsmap=e.precompiled_charsmap??null}normalize(e){return e=e.replace(/[\u0001-\u0008\u000B\u000E-\u001F\u007F\u008F\u009F]/gm,``),e=e.replace(/[\u0009\u000A\u000C\u000D\u00A0\u1680\u2000-\u200F\u2028\u2029\u202F\u205F\u2581\u3000\uFEFF\uFFFD]/gm,` `),e=e.includes(`～`)?e.split(`～`).map(e=>e.normalize(`NFKC`)).join(`～`):e.normalize(`NFKC`),e}},Jt=class extends N{constructor(e){super(e),this.normalizers=(e.normalizers??[]).map(e=>on(e))}normalize(e){return this.normalizers.reduce((e,t)=>t?t.normalize(e):e,e)}},Yt=class extends N{normalize(e){let t=A(this.config.pattern??{});return t===null?e:e.replaceAll(t,this.config.content??``)}},P=class extends N{constructor(){super(...arguments),this.form=`NFC`}normalize(e){return e=e.normalize(this.form),e}},Xt=class extends P{constructor(){super(...arguments),this.form=`NFC`}},Zt=class extends P{constructor(){super(...arguments),this.form=`NFD`}},Qt=class extends P{constructor(){super(...arguments),this.form=`NFKC`}},$t=class extends P{constructor(){super(...arguments),this.form=`NFKD`}},en=class extends N{normalize(e){return this.config.strip_left&&this.config.strip_right?e=e.trim():(this.config.strip_left&&(e=e.trimStart()),this.config.strip_right&&(e=e.trimEnd())),e}},tn=class extends N{normalize(e){return Ut(e)}},nn=class extends N{normalize(e){return e.toLowerCase()}},rn=class extends N{normalize(e){return e=this.config.prepend+e,e}};function an(e){if(e===null)return null;switch(e.type){case`BertNormalizer`:return new Kt(e);case`Precompiled`:return new qt(e);case`Sequence`:return new Jt(e);case`Replace`:return new Yt(e);case`NFC`:return new Xt(e);case`NFD`:return new Zt(e);case`NFKC`:return new Qt(e);case`NFKD`:return new $t(e);case`Strip`:return new en(e);case`StripAccents`:return new tn(e);case`Lowercase`:return new nn(e);case`Prepend`:return new rn(e);default:throw Error(`Unknown Normalizer type: ${e.type}`)}}var on=an,F=class extends M{pre_tokenize(e,t){return(Array.isArray(e)?e.map(e=>this.pre_tokenize_text(e,t)):this.pre_tokenize_text(e,t)).flat()}_call(e,t){return this.pre_tokenize(e,t)}},sn=class extends F{constructor(e){super(),this.config=e,this.add_prefix_space=this.config.add_prefix_space??!1,this.trim_offsets=this.config.trim_offsets??!1,this.use_regex=this.config.use_regex??!0,this.pattern=/'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu,this.byte_encoder=At,this.text_encoder=new TextEncoder}pre_tokenize_text(e,t){return this.add_prefix_space&&!e.startsWith(` `)&&(e=` `+e),(this.use_regex?e.match(this.pattern)||[]:[e]).map(e=>Array.from(this.text_encoder.encode(e),e=>this.byte_encoder[e]).join(``))}},cn=class extends F{pre_tokenize_text(e,t){return e.match(/\w+|[^\w\s]+/g)||[]}},ln=class extends F{constructor(e){super(),this.replacement=e.replacement??`▁`,this.str_rep=e.str_rep||this.replacement,this.prepend_scheme=e.prepend_scheme??`always`}pre_tokenize_text(e,t){let{section_index:n=void 0}=t??{},r=e.replaceAll(` `,this.str_rep);return!r.startsWith(this.replacement)&&(this.prepend_scheme===`always`||this.prepend_scheme===`first`&&n===0)&&(r=this.str_rep+r),[r]}},un=class extends F{constructor(e){super(),this.config=e,this.pattern=A(this.config.pattern??{},this.config.invert??!0)}pre_tokenize_text(e){return this.pattern===null?[]:this.config.invert?e.match(this.pattern)||[]:this.config.behavior?.toLowerCase()===`removed`?e.split(this.pattern).filter(e=>e):Ht(e,this.pattern)}},dn=class extends F{constructor(e){super(),this.config=e,this.pattern=RegExp(`[^${k}]+|[${k}]+`,`gu`)}pre_tokenize_text(e){return e.match(this.pattern)||[]}},fn=class extends F{constructor(e){super(),this.config=e;let t=`[^\\d]+|\\d${this.config.individual_digits?``:`+`}`;this.pattern=new RegExp(t,`gu`)}pre_tokenize_text(e){return e.match(this.pattern)||[]}},pn=class extends F{constructor(){super(),this.pattern=RegExp(`[^\\s${k}]+|[${k}]`,`gu`)}pre_tokenize_text(e,t){return e.trim().match(this.pattern)||[]}},mn=class extends F{constructor(e){super(),this.config=e,this.pattern=A(this.config.pattern??{}),this.content=this.config.content??``}pre_tokenize_text(e){return this.pattern===null?[e]:[e.replaceAll(this.pattern,this.config.content??``)]}},hn=class extends F{constructor(e){super(),this.tokenizers=(e.pretokenizers??[]).map(e=>yn(e))}pre_tokenize_text(e,t){return this.tokenizers.reduce((e,n)=>n?n.pre_tokenize(e,t):e,[e])}},gn=class extends F{pre_tokenize_text(e){return Gt(e)}},_n=class extends F{constructor(e){super(),this.config=e,this._length=e.length}pre_tokenize_text(e){let t=[];for(let n=0;n<e.length;n+=this._length)t.push(e.slice(n,n+this._length));return t}};function vn(e){if(e===null)return null;switch(e.type){case`BertPreTokenizer`:return new pn;case`Sequence`:return new hn(e);case`Whitespace`:return new cn;case`WhitespaceSplit`:return new gn;case`Metaspace`:return new ln(e);case`ByteLevel`:return new sn(e);case`Split`:return new un(e);case`Punctuation`:return new dn(e);case`Digits`:return new fn(e);case`Replace`:return new mn(e);case`FixedLength`:return new _n(e);default:throw Error(`Unknown PreTokenizer type: ${e.type}`)}}var yn=vn,I=class extends M{constructor(e){super(),this.config=e,this.vocab=[],this.tokens_to_ids=new Map,this.unk_token_id=void 0,this.unk_token=void 0,this.end_of_word_suffix=void 0,this.fuse_unk=this.config.fuse_unk??!1}_call(e){let t=this.encode(e);return this.fuse_unk&&(t=It(t,this.tokens_to_ids,this.unk_token_id)),t}},bn=class extends I{constructor(e){super(e),this.max_input_chars_per_word=100,this.tokens_to_ids=Vt(e.vocab),this.unk_token_id=this.tokens_to_ids.get(e.unk_token),this.unk_token=e.unk_token,this.max_input_chars_per_word=e.max_input_chars_per_word??100,this.vocab=Array(this.tokens_to_ids.size);for(let[e,t]of this.tokens_to_ids)this.vocab[t]=e}encode(e){let t=[];for(let n of e){let e=[...n];if(e.length>this.max_input_chars_per_word){t.push(this.unk_token);continue}let r=!1,i=0,a=[];for(;i<e.length;){let t=e.length,n=null;for(;i<t;){let r=e.slice(i,t).join(``);if(i>0&&(r=this.config.continuing_subword_prefix+r),this.tokens_to_ids.has(r)){n=r;break}--t}if(n===null){r=!0;break}a.push(n),i=t}r?t.push(this.unk_token):t.push(...a)}return t}},xn=class e{constructor(e,t){this.is_leaf=e,this.children=t}static default(){return new e(!1,new Map)}},Sn=class{constructor(){this.root=xn.default()}extend(e){for(let t of e)this.push(t)}push(e){let t=this.root;for(let n of e){let e=t.children.get(n);e===void 0&&(e=xn.default(),t.children.set(n,e)),t=e}t.is_leaf=!0}*common_prefix_search(e){let t=this.root;if(t===void 0)return;let n=``;for(let r of e){if(n+=r,t=t.children.get(r),t===void 0)return;t.is_leaf&&(yield n)}}},Cn=class e{constructor(e,t,n,r,i){this.token_id=e,this.node_id=t,this.pos=n,this.length=r,this.score=i,this.prev=null,this.backtrace_score=0}clone(){let t=new e(this.token_id,this.node_id,this.pos,this.length,this.score);return t.prev=this.prev,t.backtrace_score=this.backtrace_score,t}},wn=class{constructor(e,t,n){this.chars=Array.from(e),this.len=this.chars.length,this.bos_token_id=t,this.eos_token_id=n,this.nodes=[],this.begin_nodes=Array.from({length:this.len+1},()=>[]),this.end_nodes=Array.from({length:this.len+1},()=>[]);let r=new Cn(this.bos_token_id??0,0,0,0,0),i=new Cn(this.eos_token_id??0,1,this.len,0,0);this.nodes.push(r.clone()),this.nodes.push(i.clone()),this.begin_nodes[this.len].push(i),this.end_nodes[0].push(r)}insert(e,t,n,r){let i=this.nodes.length,a=new Cn(r,i,e,t,n);this.begin_nodes[e].push(a),this.end_nodes[e+t].push(a),this.nodes.push(a)}viterbi(){let e=this.len,t=0;for(;t<=e;){if(this.begin_nodes[t].length==0)return[];for(let e of this.begin_nodes[t]){e.prev=null;let n=0,r=null;for(let i of this.end_nodes[t]){let t=i.backtrace_score+e.score;(r===null||t>n)&&(r=i.clone(),n=t)}if(r!==null)e.prev=r,e.backtrace_score=n;else return[]}++t}let n=[],r=this.begin_nodes[e][0].prev;if(r===null)return[];let i=r.clone();for(;i.prev!==null;)n.push(i.clone()),i=i.clone().prev.clone();return n.reverse(),n}piece(e){return this.chars.slice(e.pos,e.pos+e.length).join(``)}tokens(){return this.viterbi().map(e=>this.piece(e))}token_ids(){return this.viterbi().map(e=>e.token_id)}};function Tn(e){if(e.length===0)throw Error(`Array must not be empty`);let t=e[0],n=0;for(let r=1;r<e.length;++r)e[r]<t&&(t=e[r],n=r);return[t,n]}var En=class extends I{constructor(e,t){super(e);let n=e.vocab.length;this.vocab=Array(n),this.scores=Array(n);for(let t=0;t<n;++t)[this.vocab[t],this.scores[t]]=e.vocab[t];this.unk_token_id=e.unk_id,this.unk_token=this.vocab[e.unk_id],this.tokens_to_ids=new Map(this.vocab.map((e,t)=>[e,t])),this.bos_token=` `,this.bos_token_id=this.tokens_to_ids.get(this.bos_token),this.eos_token=t,this.eos_token_id=this.tokens_to_ids.get(this.eos_token),this.unk_token=this.vocab[this.unk_token_id],this.min_score=Tn(this.scores)[0],this.unk_score=this.min_score-10,this.scores[this.unk_token_id]=this.unk_score,this.trie=new Sn,this.trie.extend(this.vocab),this.fuse_unk=!0}populate_nodes(e){let t=e.chars,n=0;for(;n<t.length;){let r=!1,i=[],a=t.slice(n).join(``),o=this.trie.common_prefix_search(a);for(let t of o){i.push(t);let a=this.tokens_to_ids.get(t),o=this.scores[a],s=zt(t);e.insert(n,s,o,a),!r&&s===1&&(r=!0)}r||e.insert(n,1,this.unk_score,this.unk_token_id),n+=1}}tokenize(e){let t=new wn(e,this.bos_token_id,this.eos_token_id);return this.populate_nodes(t),t.tokens()}encode(e){let t=[];for(let n of e){let e=this.tokenize(n);t.push(...e)}return t}},Dn=class{constructor(e=(e,t)=>e>t,t=1/0){this._heap=[],this._comparator=e,this._max_size=t}get size(){return this._heap.length}is_empty(){return this.size===0}peek(){return this._heap[0]}push(...e){return this.extend(e)}extend(e){for(let t of e)if(this.size<this._max_size)this._heap.push(t),this._sift_up();else{let e=this._smallest();this._comparator(t,this._heap[e])&&(this._heap[e]=t,this._sift_up_from(e))}return this.size}pop(){let e=this.peek(),t=this.size-1;return t>0&&this._swap(0,t),this._heap.pop(),this._sift_down(),e}replace(e){let t=this.peek();return this._heap[0]=e,this._sift_down(),t}_parent(e){return(e+1>>>1)-1}_left(e){return(e<<1)+1}_right(e){return e+1<<1}_greater(e,t){return this._comparator(this._heap[e],this._heap[t])}_swap(e,t){let n=this._heap[e];this._heap[e]=this._heap[t],this._heap[t]=n}_sift_up(){this._sift_up_from(this.size-1)}_sift_up_from(e){for(;e>0&&this._greater(e,this._parent(e));)this._swap(e,this._parent(e)),e=this._parent(e)}_sift_down(){let e=0;for(;this._left(e)<this.size&&this._greater(this._left(e),e)||this._right(e)<this.size&&this._greater(this._right(e),e);){let t=this._right(e)<this.size&&this._greater(this._right(e),this._left(e))?this._right(e):this._left(e);this._swap(e,t),e=t}}_smallest(){return 2**Math.floor(Math.log2(this.size))-1}},On=class{constructor(e){this.capacity=e,this.cache=new Map}get(e){if(!this.cache.has(e))return;let t=this.cache.get(e);return this.cache.delete(e),this.cache.set(e,t),t}put(e,t){this.cache.has(e)&&this.cache.delete(e),this.cache.set(e,t),this.cache.size>this.capacity&&this.cache.delete(this.cache.keys().next().value)}clear(){this.cache.clear()}},kn=class extends I{constructor(e){super(e),this.tokens_to_ids=Vt(e.vocab),this.unk_token_id=this.tokens_to_ids.get(e.unk_token),this.unk_token=e.unk_token,this.vocab=Array(this.tokens_to_ids.size);for(let[e,t]of this.tokens_to_ids)this.vocab[t]=e;let t=Array.isArray(e.merges[0]);this.merges=t?e.merges:e.merges.map(e=>e.split(` `,2)),this.bpe_ranks=new Map(this.merges.map((e,t)=>[JSON.stringify(e),t])),this.end_of_word_suffix=e.end_of_word_suffix,this.continuing_subword_suffix=e.continuing_subword_suffix??null,this.byte_fallback=this.config.byte_fallback??!1,this.byte_fallback&&(this.text_encoder=new TextEncoder),this.ignore_merges=this.config.ignore_merges??!1,this.max_length_to_cache=256,this.cache_capacity=1e4,this.cache=new On(this.cache_capacity)}clear_cache(){this.cache.clear()}bpe(e){if(e.length===0)return[];let t=this.cache.get(e);if(t!==void 0)return t;let n=Array.from(e);this.end_of_word_suffix&&(n[n.length-1]+=this.end_of_word_suffix);let r=[];if(n.length>1){let e=new Dn((e,t)=>e.score<t.score),t={token:n[0],bias:0,prev:null,next:null},i=t;for(let t=1;t<n.length;++t){let r={bias:t/n.length,token:n[t],prev:i,next:null};i.next=r,this.add_node(e,i),i=r}for(;!e.is_empty();){let n=e.pop();if(n.deleted||!n.next||n.next.deleted)continue;if(n.deleted=!0,n.next.deleted=!0,n.prev){let e={...n.prev};n.prev.deleted=!0,n.prev=e,e.prev?e.prev.next=e:t=e}let r={token:n.token+n.next.token,bias:n.bias,prev:n.prev,next:n.next.next};r.prev?(r.prev.next=r,this.add_node(e,r.prev)):t=r,r.next&&(r.next.prev=r,this.add_node(e,r))}for(let e=t;e!==null;e=e.next)r.push(e.token)}else r=n;if(this.continuing_subword_suffix)for(let e=0;e<r.length-1;++e)r[e]+=this.continuing_subword_suffix;return e.length<this.max_length_to_cache&&this.cache.put(e,r),r}add_node(e,t){let n=this.bpe_ranks.get(JSON.stringify([t.token,t.next.token]));n!==void 0&&(t.score=n+t.bias,e.push(t))}encode(e){let t=[];for(let n of e){if(this.ignore_merges&&this.tokens_to_ids.has(n)){t.push(n);continue}let e=this.bpe(n);for(let n of e)if(this.tokens_to_ids.has(n))t.push(n);else if(this.byte_fallback){let e=Array.from(this.text_encoder.encode(n)).map(e=>`<0x${e.toString(16).toUpperCase().padStart(2,`0`)}>`);e.every(e=>this.tokens_to_ids.has(e))?t.push(...e):this.unk_token!=null&&t.push(this.unk_token)}else this.unk_token!=null&&t.push(this.unk_token)}return t}},An=class extends I{constructor(e,t){super(e);let n=e.vocab;this.tokens_to_ids=Vt(t.target_lang?n[t.target_lang]:n),this.bos_token=t.bos_token,this.bos_token_id=this.tokens_to_ids.get(this.bos_token),this.eos_token=t.eos_token,this.eos_token_id=this.tokens_to_ids.get(this.eos_token),this.pad_token=t.pad_token,this.pad_token_id=this.tokens_to_ids.get(this.pad_token),this.unk_token=t.unk_token,this.unk_token_id=this.tokens_to_ids.get(this.unk_token),this.vocab=Array(this.tokens_to_ids.size);for(let[e,t]of this.tokens_to_ids)this.vocab[t]=e}encode(e){return e}};function jn(e,t){switch(e.type){case`WordPiece`:return new bn(e);case`Unigram`:return new En(e,t.eos_token);case`BPE`:return new kn(e);default:if(e.vocab)return Array.isArray(e.vocab)?new En(e,t.eos_token):Object.hasOwn(e,`continuing_subword_prefix`)&&Object.hasOwn(e,`unk_token`)?Object.hasOwn(e,`merges`)?new kn(e):new bn(e):new An(e,{target_lang:t.target_lang,bos_token:t.bos_token,eos_token:t.eos_token,pad_token:t.pad_token,unk_token:t.unk_token});throw Error(`Unknown TokenizerModel type: ${e?.type}`)}}var Mn=jn,L=class extends M{constructor(e){super(),this.config=e}_call(e,...t){return this.post_process(e,...t)}},Nn=class extends L{post_process(e,t=null,n=!0){let r=t===null?this.config.single:this.config.pair,i=[],a=[];for(let o of r)`SpecialToken`in o?n&&(i.push(o.SpecialToken.id),a.push(o.SpecialToken.type_id)):`Sequence`in o&&(o.Sequence.id===`A`?(i=j(i,e),a=j(a,Array(e.length).fill(o.Sequence.type_id))):o.Sequence.id===`B`&&(i=j(i,t),a=j(a,Array(t.length).fill(o.Sequence.type_id))));return{tokens:i,token_type_ids:a}}},Pn=class extends L{post_process(e,t=null){return{tokens:e,tokens_pair:t}}},Fn=class extends L{constructor(e){super(e),this.sep=e.sep,this.cls=e.cls}post_process(e,t=null,n=!0){n&&(e=j([this.cls[0]],e,[this.sep[0]]));let r=Array(e.length).fill(0);if(t){let i=[],a=n?[this.sep[0]]:[];e=j(e,i,t,a),r=j(r,Array(t.length+i.length+a.length).fill(1))}return{tokens:e,token_type_ids:r}}},In=class extends L{constructor(e){super(e),this.sep=e.sep,this.cls=e.cls}post_process(e,t,n=!0){n&&(e=j([this.cls[0]],e,[this.sep[0]]));let r=Array(e.length).fill(0);if(t){let i=n?[this.sep[0]]:[],a=n?[this.sep[0]]:[];e=j(e,i,t,a),r=j(r,Array(t.length+i.length+a.length).fill(1))}return{tokens:e,token_type_ids:r}}},Ln=class extends L{constructor(e){super(e),this.processors=(e.processors??[]).map(e=>zn(e))}post_process(e,t=null,n=!0){let r={tokens:e,tokens_pair:t};for(let e of this.processors)r=e.post_process(r.tokens,r.tokens_pair,n);return r}};function Rn(e){if(e===null)return null;switch(e.type){case`TemplateProcessing`:return new Nn(e);case`ByteLevel`:return new Pn(e);case`BertProcessing`:return new Fn(e);case`RobertaProcessing`:return new In(e);case`Sequence`:return new Ln(e);default:throw Error(`Unknown PostProcessor type: ${e.type}`)}}var zn=Rn,R=class extends M{constructor(e){super(),this.config=e,this.added_tokens=[],this.end_of_word_suffix=null,this.trim_offsets=`trim_offsets`in e?e.trim_offsets:!1}_call(e){return this.decode(e)}decode(e){return this.decode_chain(e).join(``)}},Bn=class extends R{constructor(e){super(e),this.byte_decoder=jt,this.text_decoder=new TextDecoder(`utf-8`,{fatal:!1,ignoreBOM:!0}),this.end_of_word_suffix=null}convert_tokens_to_string(e){let t=e.join(``),n=new Uint8Array([...t].map(e=>this.byte_decoder[e]));return this.text_decoder.decode(n)}decode_chain(e){let t=[],n=[];for(let r of e)this.added_tokens.find(e=>e.content===r)===void 0?n.push(r):(n.length>0&&(t.push(this.convert_tokens_to_string(n)),n=[]),t.push(r));return n.length>0&&t.push(this.convert_tokens_to_string(n)),t}},Vn=class extends R{constructor(e){super(e),this.cleanup=e.cleanup}decode_chain(e){return e.map((e,t)=>{if(t!==0){let t=this.config.prefix;e=t&&e.startsWith(t)?e.replace(t,``):` `+e}return this.cleanup&&(e=Pt(e)),e})}},Hn=class extends R{constructor(e){super(e),this.replacement=e.replacement??`▁`}decode_chain(e){let t=[];for(let n=0;n<e.length;++n){let r=e[n].replaceAll(this.replacement,` `);n==0&&r.startsWith(` `)&&(r=r.substring(1)),t.push(r)}return t}},Un=class extends R{constructor(e){super(e),this.suffix=e.suffix??``}decode_chain(e){return e.map((t,n)=>t.replaceAll(this.suffix,n===e.length-1?``:` `))}},Wn=class extends R{constructor(e){super(e),this.pad_token=e.pad_token??``,this.word_delimiter_token=e.word_delimiter_token??``,this.cleanup=e.cleanup}convert_tokens_to_string(e){if(e.length===0)return``;let t=[e[0]];for(let n=1;n<e.length;++n)e[n]!==t.at(-1)&&t.push(e[n]);let n=t.filter(e=>e!==this.pad_token).join(``);return this.cleanup&&(n=Pt(n).replaceAll(this.word_delimiter_token,` `).trim()),n}decode_chain(e){return[this.convert_tokens_to_string(e)]}},Gn=class extends R{constructor(e){super(e),this.decoders=(e.decoders??[]).map(e=>Zn(e))}decode_chain(e){return this.decoders.reduce((e,t)=>t.decode_chain(e),e)}},Kn=class extends R{decode_chain(e){let t=A(this.config.pattern),n=this.config.content??``;return t===null?e:e.map(e=>e.replaceAll(t,n))}},qn=class extends R{decode_chain(e){return[e.join(``)]}},Jn=class extends R{constructor(e){super(e),this.content=e.content??``,this.start=e.start??0,this.stop=e.stop??0}decode_chain(e){return e.map(e=>{let t=0;for(let n=0;n<this.start&&e[n]===this.content;++n)t=n+1;let n=e.length;for(let t=0;t<this.stop;++t){let r=e.length-t-1;if(e[r]===this.content){n=r;continue}else break}return e.slice(t,n)})}},Yn=class extends R{constructor(e){super(e),this.text_decoder=new TextDecoder}decode_chain(e){let t=[],n=[];for(let r of e){let e=null;if(r.length===6&&r.startsWith(`<0x`)&&r.endsWith(`>`)){let t=parseInt(r.slice(3,5),16);isNaN(t)||(e=t)}if(e!==null)n.push(e);else{if(n.length>0){let e=this.text_decoder.decode(Uint8Array.from(n));t.push(e),n=[]}t.push(r)}}if(n.length>0){let e=this.text_decoder.decode(Uint8Array.from(n));t.push(e),n=[]}return t}};function Xn(e){if(e===null)return null;switch(e.type){case`ByteLevel`:return new Bn(e);case`WordPiece`:return new Vn(e);case`Metaspace`:return new Hn(e);case`BPEDecoder`:return new Un(e);case`CTC`:return new Wn(e);case`Sequence`:return new Gn(e);case`Replace`:return new Kn(e);case`Fuse`:return new qn(e);case`Strip`:return new Jn(e);case`ByteFallback`:return new Yn(e);default:throw Error(`Unknown Decoder type: ${e.type}`)}}var Zn=Xn,Qn=class{constructor(e,t){let n=Wt(e,`Tokenizer`,[`model`,`decoder`,`post_processor`,`pre_tokenizer`,`normalizer`]);if(n)throw Error(n);let r=Wt(t,`Config`);if(r)throw Error(r);this.tokenizer=e,this.config=t,this.normalizer=on(this.tokenizer.normalizer),this.pre_tokenizer=yn(this.tokenizer.pre_tokenizer),this.model=Mn(this.tokenizer.model,this.config),this.post_processor=zn(this.tokenizer.post_processor),this.decoder=Zn(this.tokenizer.decoder),this.special_tokens=[],this.all_special_ids=[],this.added_tokens=[];let i=[],a=[];this.added_tokens_map=new Map;for(let e of this.tokenizer.added_tokens){let t=new kt(e);if(this.added_tokens.push(t),this.model.tokens_to_ids.set(t.content,t.id),this.model.vocab[t.id]=t.content,t.special&&(this.special_tokens.push(t.content),this.all_special_ids.push(t.id)),this.added_tokens_map.set(t.content,t),t.normalized&&this.normalizer!==null){let e=this.normalizer(t.content);a.push(e),this.added_tokens_map.set(e,t)}else i.push(t.content)}(this.config.additional_special_tokens??[]).forEach(e=>{this.special_tokens.includes(e)||this.special_tokens.push(e)}),this.decoder&&(this.decoder.added_tokens=this.added_tokens,this.decoder.end_of_word_suffix=this.model.end_of_word_suffix),this.splitter_unnormalized=new Ot(i),this.splitter_normalized=new Ot(a),this.remove_space=this.config.remove_space,this.clean_up_tokenization_spaces=this.config.clean_up_tokenization_spaces??!0,this.do_lowercase_and_remove_accent=this.config.do_lowercase_and_remove_accent??!1}encode(e,{text_pair:t=null,add_special_tokens:n=!0,return_token_type_ids:r=null}={}){let{tokens:i,token_type_ids:a}=this.tokenize_helper(e,{text_pair:t,add_special_tokens:n}),o=i.map(e=>this.added_tokens_map.get(e)?.id??this.model.tokens_to_ids.get(e)??this.model.unk_token_id),s={ids:o,tokens:i,attention_mask:Array(o.length).fill(1)};return r&&a&&(s.token_type_ids=a),s}decode(e,t={}){if(!Array.isArray(e)||e.length===0||!Rt(e[0]))throw Error(`token_ids must be a non-empty array of integers.`);let n=e.map(e=>this.model.vocab[Number(e)]??this.model.unk_token);t.skip_special_tokens&&(n=n.filter(e=>!this.special_tokens.includes(e)));let r=this.decoder?this.decoder(n):n.join(` `);return this.decoder&&this.decoder.end_of_word_suffix&&(r=r.replaceAll(this.decoder.end_of_word_suffix,` `),t.skip_special_tokens&&(r=r.trim())),(t.clean_up_tokenization_spaces??this.clean_up_tokenization_spaces)&&(r=Pt(r)),r}tokenize(e,{text_pair:t=null,add_special_tokens:n=!1}={}){return this.tokenize_helper(e,{text_pair:t,add_special_tokens:n}).tokens}encode_text(e){if(e===null)return null;let t=this.splitter_unnormalized.split(e);return t.forEach((e,n)=>{let r=this.added_tokens_map.get(e);r&&(r.lstrip&&n>0&&(t[n-1]=t[n-1].trimEnd()),r.rstrip&&n<t.length-1&&(t[n+1]=t[n+1].trimStart()))}),t.flatMap((e,t)=>{if(e.length===0)return[];if(this.added_tokens_map.has(e))return[e];if(this.remove_space===!0&&(e=e.trim().split(/\s+/).join(` `)),this.do_lowercase_and_remove_accent&&(e=Bt(e)),this.normalizer!==null&&(e=this.normalizer(e)),e.length===0)return[];let n=this.splitter_normalized.split(e);return n.forEach((e,t)=>{let r=this.added_tokens_map.get(e);r&&(r.lstrip&&t>0&&(n[t-1]=n[t-1].trimEnd()),r.rstrip&&t<n.length-1&&(n[t+1]=n[t+1].trimStart()))}),n.flatMap(e=>{if(e.length===0)return[];if(this.added_tokens_map.has(e))return[e];let n=this.pre_tokenizer===null?[e]:this.pre_tokenizer(e,{section_index:t});return this.model(n)})})}tokenize_helper(e,{text_pair:t=null,add_special_tokens:n=!0}){let r=this.encode_text(e),i=this.encode_text(t||null);return this.post_processor?this.post_processor(r,i,n):{tokens:j(r??[],i??[])}}token_to_id(e){return this.model.tokens_to_ids.get(e)}id_to_token(e){return this.model.vocab[e]}get_added_tokens_decoder(){let e=new Map;for(let t of this.added_tokens)e.set(t.id,t);return e}get_vocab(e=!0){let t=new Map;for(let n=0;n<this.model.vocab.length;++n){let r=this.model.vocab[n];(e||!this.added_tokens_map.has(r))&&t.set(r,n)}return t}};const $n=(e,t)=>{if(!e.has(t))throw Error(`Missing required key ${JSON.stringify(t)} in structured data.`);return e.get(t)},z=e=>{if(Array.isArray(e))return e.map(e=>z(e));if(e instanceof Map){let t=Object.create(null);for(let[n,r]of e.entries()){if(typeof n!=`string`)throw TypeError(`Expected string structured-data key, got ${typeof n}.`);t[n]=z(r)}return t}return e},er=Qn,tr={add_special_tokens:!1},nr={clean_up_tokenization_spaces:!1,skip_special_tokens:!1};var rr=class extends gt{modelId;constructor(e){super(),this.modelId=e}createState(){return{tokenizer:new er(z(O(this.modelId,`tokenizer.msgpack`)),z(O(this.modelId,`tokenizer_config.msgpack`)))}}encodeWithState(e,t){return[...t.tokenizer.encode(e,tr).ids]}tokenizeWithState(e,t){let n=this.encodeWithState(e,t);return mt(n,n.map(e=>ft(t.tokenizer.decode([e],nr))))}},ir=e({Tiktoken:()=>yr,__wbg_parse_def2e24ef1252aff:()=>br,__wbg_set_wasm:()=>ar,__wbg_stringify_f7ed6987935b4a24:()=>xr,__wbindgen_error_new:()=>Sr,__wbindgen_is_undefined:()=>Cr,__wbindgen_object_drop_ref:()=>wr,__wbindgen_string_get:()=>Tr,__wbindgen_throw:()=>Er});let B;function ar(e){B=e}let or=new(typeof TextDecoder>`u`?(0,module.require)(`util`).TextDecoder:TextDecoder)(`utf-8`,{ignoreBOM:!0,fatal:!0});or.decode();let V=null;function H(){return(V===null||V.byteLength===0)&&(V=new Uint8Array(B.memory.buffer)),V}function U(e,t){return e>>>=0,or.decode(H().subarray(e,e+t))}const W=Array(128).fill(void 0);W.push(void 0,null,!0,!1);let G=W.length;function K(e){G===W.length&&W.push(W.length+1);let t=G;return G=W[t],W[t]=e,t}function sr(e,t){try{return e.apply(this,t)}catch(e){B.__wbindgen_export_0(K(e))}}function cr(e){return W[e]}function lr(e){e<132||(W[e]=G,G=e)}function q(e){let t=cr(e);return lr(e),t}let J=0,ur=new(typeof TextEncoder>`u`?(0,module.require)(`util`).TextEncoder:TextEncoder)(`utf-8`);const dr=typeof ur.encodeInto==`function`?function(e,t){return ur.encodeInto(e,t)}:function(e,t){let n=ur.encode(e);return t.set(n),{read:e.length,written:n.length}};function Y(e,t,n){if(n===void 0){let n=ur.encode(e),r=t(n.length,1)>>>0;return H().subarray(r,r+n.length).set(n),J=n.length,r}let r=e.length,i=t(r,1)>>>0,a=H(),o=0;for(;o<r;o++){let t=e.charCodeAt(o);if(t>127)break;a[i+o]=t}if(o!==r){o!==0&&(e=e.slice(o)),i=n(i,r,r=o+e.length*3,1)>>>0;let t=H().subarray(i+o,i+r),a=dr(e,t);o+=a.written,i=n(i,r,o,1)>>>0}return J=o,i}function fr(e){return e==null}let X=null;function Z(){return(X===null||X.buffer.detached===!0||X.buffer.detached===void 0&&X.buffer!==B.memory.buffer)&&(X=new DataView(B.memory.buffer)),X}let Q=null;function pr(){return(Q===null||Q.byteLength===0)&&(Q=new Uint32Array(B.memory.buffer)),Q}function mr(e,t){return e>>>=0,pr().subarray(e/4,e/4+t)}function hr(e,t){let n=t(e.length*1,1)>>>0;return H().set(e,n/1),J=e.length,n}function gr(e,t){let n=t(e.length*4,4)>>>0;return pr().set(e,n/4),J=e.length,n}function _r(e,t){return e>>>=0,H().subarray(e/1,e/1+t)}const vr=typeof FinalizationRegistry>`u`?{register:()=>{},unregister:()=>{}}:new FinalizationRegistry(e=>B.__wbg_tiktoken_free(e>>>0,1));var yr=class{__destroy_into_raw(){let e=this.__wbg_ptr;return this.__wbg_ptr=0,vr.unregister(this),e}free(){if(B==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);let e=this.__destroy_into_raw();B.__wbg_tiktoken_free(e,0)}constructor(e,t,n){if(B==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);let r=Y(e,B.__wbindgen_export_1,B.__wbindgen_export_2),i=J,a=Y(n,B.__wbindgen_export_1,B.__wbindgen_export_2),o=J,s=B.tiktoken_new(r,i,K(t),a,o);return this.__wbg_ptr=s>>>0,vr.register(this,this.__wbg_ptr,this),this}get name(){try{let n=B.__wbindgen_add_to_stack_pointer(-16);B.tiktoken_name(n,this.__wbg_ptr);var e=Z().getInt32(n+0,!0),t=Z().getInt32(n+4,!0);let r;return e!==0&&(r=U(e,t).slice(),B.__wbindgen_export_3(e,t*1,1)),r}finally{B.__wbindgen_add_to_stack_pointer(16)}}encode(e,t,n){if(B==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);try{let s=B.__wbindgen_add_to_stack_pointer(-16),c=Y(e,B.__wbindgen_export_1,B.__wbindgen_export_2),l=J;B.tiktoken_encode(s,this.__wbg_ptr,c,l,K(t),K(n));var r=Z().getInt32(s+0,!0),i=Z().getInt32(s+4,!0),a=Z().getInt32(s+8,!0);if(Z().getInt32(s+12,!0))throw q(a);var o=mr(r,i).slice();return B.__wbindgen_export_3(r,i*4,4),o}finally{B.__wbindgen_add_to_stack_pointer(16)}}encode_ordinary(e){if(B==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);try{let i=B.__wbindgen_add_to_stack_pointer(-16),a=Y(e,B.__wbindgen_export_1,B.__wbindgen_export_2),o=J;B.tiktoken_encode_ordinary(i,this.__wbg_ptr,a,o);var t=Z().getInt32(i+0,!0),n=Z().getInt32(i+4,!0),r=mr(t,n).slice();return B.__wbindgen_export_3(t,n*4,4),r}finally{B.__wbindgen_add_to_stack_pointer(16)}}encode_with_unstable(e,t,n){if(B==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);try{let a=B.__wbindgen_add_to_stack_pointer(-16),o=Y(e,B.__wbindgen_export_1,B.__wbindgen_export_2),s=J;B.tiktoken_encode_with_unstable(a,this.__wbg_ptr,o,s,K(t),K(n));var r=Z().getInt32(a+0,!0),i=Z().getInt32(a+4,!0);if(Z().getInt32(a+8,!0))throw q(i);return q(r)}finally{B.__wbindgen_add_to_stack_pointer(16)}}encode_single_token(e){if(B==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);let t=hr(e,B.__wbindgen_export_1),n=J;return B.tiktoken_encode_single_token(this.__wbg_ptr,t,n)>>>0}decode(e){if(B==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);try{let i=B.__wbindgen_add_to_stack_pointer(-16),a=gr(e,B.__wbindgen_export_1),o=J;B.tiktoken_decode(i,this.__wbg_ptr,a,o);var t=Z().getInt32(i+0,!0),n=Z().getInt32(i+4,!0),r=_r(t,n).slice();return B.__wbindgen_export_3(t,n*1,1),r}finally{B.__wbindgen_add_to_stack_pointer(16)}}decode_single_token_bytes(e){if(B==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);try{let i=B.__wbindgen_add_to_stack_pointer(-16);B.tiktoken_decode_single_token_bytes(i,this.__wbg_ptr,e);var t=Z().getInt32(i+0,!0),n=Z().getInt32(i+4,!0),r=_r(t,n).slice();return B.__wbindgen_export_3(t,n*1,1),r}finally{B.__wbindgen_add_to_stack_pointer(16)}}token_byte_values(){if(B==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);return q(B.tiktoken_token_byte_values(this.__wbg_ptr))}};function br(){return sr(function(e,t){return K(JSON.parse(U(e,t)))},arguments)}function xr(){return sr(function(e){return K(JSON.stringify(cr(e)))},arguments)}function Sr(e,t){return K(Error(U(e,t)))}function Cr(e){return cr(e)===void 0}function wr(e){q(e)}function Tr(e,t){if(B==null)throw Error(`tiktoken: WASM binary has not been propery initialized.`);let n=cr(t),r=typeof n==`string`?n:void 0;var i=fr(r)?0:Y(r,B.__wbindgen_export_1,B.__wbindgen_export_2),a=J;Z().setInt32(e+4,a,!0),Z().setInt32(e+0,i,!0)}function Er(e,t){throw Error(U(e,t))}let Dr=!1;async function Or(e){if(Dr)return ir;let t=await e({"./tiktoken_bg.js":ir}),n=`instance`in t&&t.instance instanceof WebAssembly.Instance?t.instance:t instanceof WebAssembly.Instance?t:null;if(n==null)throw Error(`Missing instance`);return ar(n.exports),Dr=!0,ir}const kr=new URL(`../tiktoken_bg.wasm`,import.meta.url).href;await(async()=>{await Or(async e=>{let t=e,n=await fetch(kr);if(!n.ok)throw Error(`Failed to load tiktoken WASM from ${JSON.stringify(kr)}: ${n.status} ${n.statusText}`);if(typeof WebAssembly.instantiateStreaming==`function`)try{return await WebAssembly.instantiateStreaming(n.clone(),t)}catch{}return WebAssembly.instantiate(await n.arrayBuffer(),t)})})();const Ar=e=>{let t=O(e,`config.msgpack`);return new yr(Qe(e,`tiktoken.model`),z($n(t,`special_tokens`)),$n(t,`pat_str`))};var jr=class extends gt{disposeState(e){e.free()}tokenizeWithState(e,t){let n=this.encodeWithState(e,t);return mt(n,n.map(e=>t.decode_single_token_bytes(e).length))}},Mr=class extends jr{modelId;constructor(e){super(),this.modelId=e}createState(){return Ar(this.modelId)}encodeWithState(e,t){return[...t.encode(e)]}};const $=new Map,Nr=e=>{let t=$.get(e);if(t)return t;let n=ke[e],r;switch(n.kind){case`tiktoken-builtin`:r=new Mr(e);break;case`tiktoken-custom`:r=new Mr(e);break;case`huggingface`:r=new rr(e);break;case`clip-bpe`:r=new Dt(e);break}return $.set(e,r),r},Pr=e=>{if(e){$.get(e)?.free(),$.delete(e);return}for(let e of $.values())e.free();$.clear()},Fr=(e,t)=>{if(typeof t==`string`){let[e]=st(t);return e}let n=t?.model;if(typeof n!=`string`)throw TypeError(`${e}() requires a single model ID as the second argument.`);let[r]=st(n);return r};function Ir(e,t){return Nr(Fr(`tokenizeLoaded`,t)).tokenize(e)}function Lr(e,t){return Nr(Fr(`countLoaded`,t)).getTokenCount(e)}async function Rr(e){let t=st(e);return await Ye(t),typeof e==`string`?t[0]:t}async function zr(e,t){let n=Fr(`tokenize`,t);return await Rr(n),Ir(e,n)}async function Br(e,t){let n=Fr(`count`,t);return await Rr(n),Lr(e,n)}const Vr=e=>{Ze(e),Pr(e)};export{zr as a,ke as c,Rr as i,Lr as n,Ir as o,Vr as r,E as s,Br as t};

package/lib/tiktoken.d.ts

@@ -1,4 +1,2 @@
-import type { TiktokenEncoding } from 'tiktoken';
-import { get_encoding, Tiktoken } from 'tiktoken';
-export { get_encoding, Tiktoken };
-export type { TiktokenEncoding };
+import { Tiktoken } from 'tiktoken';
+export { Tiktoken };

package/lib/tokenizers/TiktokenTokenizer.d.ts

@@ -5,13 +5,7 @@ declare abstract class BaseTiktokenTokenizer extends BaseTokenizer<Tiktoken> {
     protected disposeState(state: Tiktoken): void;
     protected tokenizeWithState(text: string, state: Tiktoken): import("../tokenization.js").RawTokenizeResult<string>;
 }
-export declare class BuiltinTiktokenTokenizer extends BaseTiktokenTokenizer {
-    readonly modelId: ModelId;
-    constructor(modelId: ModelId);
-    protected createState(): Tiktoken;
-    protected encodeWithState(text: string, state: Tiktoken): number[];
-}
-export declare class CustomTiktokenTokenizer extends BaseTiktokenTokenizer {
+export declare class TiktokenTokenizer extends BaseTiktokenTokenizer {
     readonly modelId: ModelId;
     constructor(modelId: ModelId);
     protected createState(): Tiktoken;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "token-vocabs",
-  "version": "0.3.0",
+  "version": "0.5.0",
   "type": "module",
   "description": "Count and inspect token IDs across several modern tokenizer families offline.",
   "keywords": [