token-vocabs 0.0.1 → 0.2.2

package/lib/modelAssets.d.ts

@@ -0,0 +1,6 @@
+export type EncodedModelAssetFiles = Record<string, string>;
+export type ModelAssetFileContent = Uint8Array | string;
+export type ModelAssetFiles = Record<string, ModelAssetFileContent>;
+export declare const isCompressedMsgpackFile: (fileName: string) => boolean;
+export declare const normalizeModelAssetFileName: (fileName: string) => string;
+export declare const prepareEncodedModelAssets: (files: EncodedModelAssetFiles) => Promise<ModelAssetFiles>;

package/lib/modelSelection.d.ts

@@ -0,0 +1,4 @@
+import type { ModelId } from './models.js';
+export type Arrayable<T> = ReadonlyArray<T> | T;
+export type ModelSelection = Arrayable<ModelId>;
+export declare const normalizeModelList: (model?: ModelSelection) => ModelId[];

package/lib/models.d.ts

@@ -0,0 +1,124 @@
+export type ModelId = 'deepseek' | 'gemma' | 'glm' | 'gpt' | 'kimi' | 'mimo' | 'minimax' | 'qwen' | 'sdxl';
+export type ModelDefinition = BuiltinTiktokenModelDefinition | ClipBpeModelDefinition | CustomTiktokenModelDefinition | HuggingFaceModelDefinition;
+type BaseModelDefinition = {
+    openrouter?: string;
+    title: string;
+};
+type BuiltinTiktokenModelDefinition = BaseModelDefinition & {
+    encoding: 'o200k_base';
+    kind: 'tiktoken-builtin';
+    source: {
+        encodingJsonUrl: string;
+    };
+};
+type CustomTiktokenModelDefinition = BaseModelDefinition & {
+    kind: 'tiktoken-custom';
+    source: {
+        modelUrl: string;
+        tokenizerConfigUrl: string;
+        tokenizerImplementationUrl: string;
+    };
+};
+type HuggingFaceModelDefinition = BaseModelDefinition & {
+    kind: 'huggingface';
+    source: {
+        tokenizerConfigUrl: string;
+        tokenizerJsonUrl: string;
+    };
+};
+type ClipBpeModelDefinition = BaseModelDefinition & {
+    kind: 'clip-bpe';
+    source: {
+        mergesUrl: string;
+        specialTokensMapUrl?: string;
+        tokenizerConfigUrl: string;
+        vocabUrl: string;
+    };
+};
+export declare const models: {
+    readonly gpt: {
+        readonly encoding: "o200k_base";
+        readonly kind: "tiktoken-builtin";
+        readonly openrouter: "openai/gpt-5.5";
+        readonly source: {
+            readonly encodingJsonUrl: "https://tiktoken.pages.dev/js/o200k_base.json";
+        };
+        readonly title: "GPT-5.5";
+    };
+    readonly gemma: {
+        readonly kind: "huggingface";
+        readonly openrouter: "google/gemma-4-31b-it";
+        readonly source: {
+            readonly tokenizerConfigUrl: "https://huggingface.co/google/gemma-4-31B-it/resolve/main/tokenizer_config.json";
+            readonly tokenizerJsonUrl: "https://huggingface.co/google/gemma-4-31B-it/resolve/main/tokenizer.json";
+        };
+        readonly title: "Gemma 4 31B it";
+    };
+    readonly qwen: {
+        readonly kind: "huggingface";
+        readonly openrouter: "qwen/qwen3.6-27b";
+        readonly source: {
+            readonly tokenizerConfigUrl: "https://huggingface.co/Qwen/Qwen3.6-27B/resolve/main/tokenizer_config.json";
+            readonly tokenizerJsonUrl: "https://huggingface.co/Qwen/Qwen3.6-27B/resolve/main/tokenizer.json";
+        };
+        readonly title: "Qwen 3.6 27B";
+    };
+    readonly kimi: {
+        readonly kind: "tiktoken-custom";
+        readonly openrouter: "moonshotai/kimi-k2.6";
+        readonly source: {
+            readonly modelUrl: "https://huggingface.co/moonshotai/Kimi-K2.6/resolve/main/tiktoken.model";
+            readonly tokenizerConfigUrl: "https://huggingface.co/moonshotai/Kimi-K2.6/resolve/main/tokenizer_config.json";
+            readonly tokenizerImplementationUrl: "https://huggingface.co/moonshotai/Kimi-K2.6/resolve/main/tokenization_kimi.py";
+        };
+        readonly title: "Kimi K2.6";
+    };
+    readonly deepseek: {
+        readonly kind: "huggingface";
+        readonly openrouter: "deepseek/deepseek-v4-pro";
+        readonly source: {
+            readonly tokenizerConfigUrl: "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/resolve/main/tokenizer_config.json";
+            readonly tokenizerJsonUrl: "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/resolve/main/tokenizer.json";
+        };
+        readonly title: "DeepSeek V4 Pro";
+    };
+    readonly mimo: {
+        readonly kind: "huggingface";
+        readonly openrouter: "xiaomi/mimo-v2.5-pro";
+        readonly source: {
+            readonly tokenizerConfigUrl: "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro/resolve/main/tokenizer_config.json";
+            readonly tokenizerJsonUrl: "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro/resolve/main/tokenizer.json";
+        };
+        readonly title: "MiMo V2.5 Pro";
+    };
+    readonly sdxl: {
+        readonly kind: "clip-bpe";
+        readonly source: {
+            readonly mergesUrl: "https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/resolve/main/tokenizer_2/merges.txt";
+            readonly specialTokensMapUrl: "https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/resolve/main/tokenizer_2/special_tokens_map.json";
+            readonly tokenizerConfigUrl: "https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/resolve/main/tokenizer_2/tokenizer_config.json";
+            readonly vocabUrl: "https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/resolve/main/tokenizer_2/vocab.json";
+        };
+        readonly title: "Stable Diffusion XL";
+    };
+    readonly glm: {
+        readonly kind: "huggingface";
+        readonly openrouter: "zai-org/glm-5.1";
+        readonly source: {
+            readonly tokenizerConfigUrl: "https://huggingface.co/zai-org/GLM-5.1/resolve/main/tokenizer_config.json";
+            readonly tokenizerJsonUrl: "https://huggingface.co/zai-org/GLM-5.1/resolve/main/tokenizer.json";
+        };
+        readonly title: "GLM 5.1";
+    };
+    readonly minimax: {
+        readonly kind: "huggingface";
+        readonly openrouter: "minimax/minimax-m2.7";
+        readonly source: {
+            readonly tokenizerConfigUrl: "https://huggingface.co/MiniMaxAI/MiniMax-M2.7/resolve/main/tokenizer_config.json";
+            readonly tokenizerJsonUrl: "https://huggingface.co/MiniMaxAI/MiniMax-M2.7/resolve/main/tokenizer.json";
+        };
+        readonly title: "MiniMax M2.7";
+    };
+};
+export declare const modelIds: Array<ModelId>;
+export {};

package/lib/structuredData.d.ts

@@ -0,0 +1,2 @@
+export declare const getRequiredMapValue: <T>(value: Map<string, unknown>, key: string) => T;
+export declare const toPlainObject: (value: unknown) => unknown;

package/lib/tokenizers/ClipTokenizer.d.ts

@@ -0,0 +1,23 @@
+import type { ModelId } from '../models.js';
+import { BaseTokenizer } from './base/BaseTokenizer.js';
+type ClipTokenizerState = {
+    byteEncoder: Array<string>;
+    mergeRanks: Map<string, number>;
+    specialTokenIds: Map<string, number>;
+    unknownTokenId: number;
+    vocabulary: Map<string, number>;
+};
+export declare class ClipTokenizer extends BaseTokenizer<ClipTokenizerState> {
+    #private;
+    readonly modelId: ModelId;
+    constructor(modelId: ModelId);
+    protected createState(): {
+        byteEncoder: string[];
+        mergeRanks: Map<string, number>;
+        specialTokenIds: Map<string, number>;
+        unknownTokenId: number;
+        vocabulary: Map<string, number>;
+    };
+    protected encodeWithState(text: string, state: ClipTokenizerState): number[];
+}
+export {};

package/lib/tokenizers/HuggingFaceTokenizer.d.ts

@@ -0,0 +1,20 @@
+import type { ModelId } from '../models.js';
+import { BaseTokenizer } from './base/BaseTokenizer.js';
+type HfEncodeResult = {
+    ids: Array<number>;
+};
+type HfTokenizer = {
+    encode: (text: string) => HfEncodeResult;
+};
+type TokenizerState = {
+    tokenizer: HfTokenizer;
+};
+export declare class HuggingFaceTokenizer extends BaseTokenizer<TokenizerState> {
+    readonly modelId: ModelId;
+    constructor(modelId: ModelId);
+    protected createState(): {
+        tokenizer: HfTokenizer;
+    };
+    protected encodeWithState(text: string, state: TokenizerState): number[];
+}
+export {};

package/lib/tokenizers/TiktokenTokenizer.d.ts

@@ -0,0 +1,15 @@
+import type { ModelId } from '../models.js';
+import { Tiktoken } from 'tiktoken';
+import { BaseTokenizer } from './base/BaseTokenizer.js';
+export declare class BuiltinTiktokenTokenizer extends BaseTokenizer<Tiktoken> {
+    readonly modelId: ModelId;
+    constructor(modelId: ModelId);
+    protected createState(): Tiktoken;
+    protected encodeWithState(text: string, state: Tiktoken): number[];
+}
+export declare class CustomTiktokenTokenizer extends BaseTokenizer<Tiktoken> {
+    readonly modelId: ModelId;
+    constructor(modelId: ModelId);
+    protected createState(): Tiktoken;
+    protected encodeWithState(text: string, state: Tiktoken): number[];
+}

package/lib/tokenizers/base/BaseTokenizer.d.ts

@@ -0,0 +1,8 @@
+export declare abstract class BaseTokenizer<TState> {
+    #private;
+    protected abstract createState(): TState;
+    encode(text: string): number[];
+    protected abstract encodeWithState(text: string, state: TState): Array<number>;
+    protected getState(): TState;
+    getTokenCount(text: string): number;
+}

package/lib/tokenizers/index.d.ts

@@ -0,0 +1,7 @@
+import type { ModelId } from '../models.js';
+type TokenizerLike = {
+    encode: (text: string) => Array<number>;
+    getTokenCount: (text: string) => number;
+};
+export declare const getTokenizer: (modelId: ModelId) => TokenizerLike;
+export {};

package/main.d.ts

@@ -0,0 +1,10 @@
+import type {ModelId, ModelSelection} from './lib/api.js'
+
+export {countTokens, modelIds, models, tokenize} from './lib/api.js'
+export {default} from './lib/api.js'
+export type {CountTokensOptions, CountTokensResult, ModelId, ModelSelection, TokenizeResult} from './lib/api.js'
+
+export declare const isModelLoaded: (modelId: ModelId) => boolean
+export declare const getLoadedModelIds: () => Array<ModelId>
+export declare const loadModel: (modelId: ModelId) => Promise<ModelId>
+export declare const loadModels: (model?: ModelSelection) => Promise<Array<ModelId>>

package/main.js

@@ -0,0 +1 @@
+import{a as e,i as t,l as n,n as r,o as i,r as a,t as o,u as s}from"./chunks/main.js";export{e as countTokens,e as default,o as getLoadedModelIds,r as isModelLoaded,a as loadModel,t as loadModels,n as modelIds,s as models,i as tokenize};

package/package.json

@@ -1 +1,47 @@
-{"name":"token-vocabs","description":"returns `'token-vocabs'`","version":"0.0.1","type":"module","exports":{".":{"types":"./index.d.ts","import":"./index.js","default":"./index.js"}},"author":"Jaid <6216144+Jaid@users.noreply.github.com> (https://github.com/Jaid)","license":"MIT","repository":"github:Jaid/token-vocabs"}
+{
+  "name": "token-vocabs",
+  "version": "0.2.2",
+  "type": "module",
+  "description": "Count and inspect token IDs across several modern tokenizer families offline.",
+  "keywords": [
+    "bpe",
+    "count-tokens",
+    "deepseek",
+    "gemma",
+    "gpt",
+    "kimi",
+    "llm",
+    "minimax",
+    "qwen",
+    "sdxl",
+    "tokenizer"
+  ],
+  "funding": "https://github.com/sponsors/Jaid",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Jaid/token-vocabs.git"
+  },
+  "homepage": "https://github.com/Jaid/token-vocabs#readme",
+  "bugs": {
+    "url": "https://github.com/Jaid/token-vocabs/issues"
+  },
+  "license": "MIT",
+  "exports": {
+    ".": {
+      "types": "./all.d.ts",
+      "import": "./all.js",
+      "default": "./all.js"
+    },
+    "./browser": {
+      "types": "./main.d.ts",
+      "import": "./main.js",
+      "default": "./main.js"
+    },
+    "./browser/all": {
+      "types": "./all.d.ts",
+      "import": "./all.js",
+      "default": "./all.js"
+    }
+  },
+  "types": "./all.d.ts"
+}