token-vocabs 0.2.2 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/README.md +80 -61
  2. package/all.d.ts +2 -2
  3. package/all.js +1 -1
  4. package/chunks/decompress.js +1 -1
  5. package/chunks/main.js +2 -2
  6. package/deepseek.bin +0 -0
  7. package/gemma.bin +0 -0
  8. package/glm.bin +0 -0
  9. package/gpt.bin +1 -0
  10. package/hy.bin +0 -0
  11. package/kimi.bin +0 -0
  12. package/lib/api.d.ts +24 -14
  13. package/lib/data.d.ts +4 -5
  14. package/lib/modelAssets/base/ModelAssetBundleLoader.d.ts +6 -0
  15. package/lib/models.d.ts +21 -1
  16. package/lib/tiktoken.d.ts +4 -0
  17. package/lib/tokenization.d.ts +38 -0
  18. package/lib/tokenizers/ClipTokenizer.d.ts +2 -0
  19. package/lib/tokenizers/HuggingFaceTokenizer.d.ts +10 -1
  20. package/lib/tokenizers/TiktokenTokenizer.d.ts +8 -3
  21. package/lib/tokenizers/base/BaseTokenizer.d.ts +7 -2
  22. package/lib/tokenizers/index.d.ts +6 -2
  23. package/main.d.ts +2 -9
  24. package/main.js +1 -1
  25. package/mimo.bin +0 -0
  26. package/minimax.bin +0 -0
  27. package/package.json +5 -5
  28. package/qwen.bin +0 -0
  29. package/sdxl.bin +0 -0
  30. package/step.bin +0 -0
  31. package/chunks/deepseek.js +0 -1
  32. package/chunks/gemma.js +0 -1
  33. package/chunks/glm.js +0 -1
  34. package/chunks/gpt.js +0 -1
  35. package/chunks/kimi.js +0 -1
  36. package/chunks/mimo.js +0 -1
  37. package/chunks/minimax.js +0 -1
  38. package/chunks/qwen.js +0 -1
  39. package/chunks/sdxl.js +0 -1
  40. package/lib/base85Decode.d.ts +0 -2
  41. package/lib/modelAssets.d.ts +0 -6
  42. package/vocabulary/deepseek.js +0 -1
  43. package/vocabulary/gemma.js +0 -1
  44. package/vocabulary/glm.js +0 -1
  45. package/vocabulary/gpt.js +0 -1
  46. package/vocabulary/kimi.js +0 -1
  47. package/vocabulary/mimo.js +0 -1
  48. package/vocabulary/minimax.js +0 -1
  49. package/vocabulary/qwen.js +0 -1
  50. package/vocabulary/sdxl.js +0 -1
  51. /package/chunks/{rolldown-runtime.js → chunk.js} +0 -0
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "token-vocabs",
3
- "version": "0.2.2",
3
+ "version": "0.4.0",
4
4
  "type": "module",
5
5
  "description": "Count and inspect token IDs across several modern tokenizer families offline.",
6
6
  "keywords": [
@@ -28,9 +28,9 @@
28
28
  "license": "MIT",
29
29
  "exports": {
30
30
  ".": {
31
- "types": "./all.d.ts",
32
- "import": "./all.js",
33
- "default": "./all.js"
31
+ "types": "./main.d.ts",
32
+ "import": "./main.js",
33
+ "default": "./main.js"
34
34
  },
35
35
  "./browser": {
36
36
  "types": "./main.d.ts",
@@ -43,5 +43,5 @@
43
43
  "default": "./all.js"
44
44
  }
45
45
  },
46
- "types": "./all.d.ts"
46
+ "types": "./main.d.ts"
47
47
  }
package/qwen.bin ADDED
Binary file
package/sdxl.bin ADDED
Binary file
package/step.bin ADDED
Binary file
package/chunks/deepseek.js DELETED
@@ -1 +0,0 @@
1
- import{t as e}from"../vocabulary/deepseek.js";export{e as default};
package/chunks/gemma.js DELETED
@@ -1 +0,0 @@
1
- import{t as e}from"../vocabulary/gemma.js";export{e as default};
package/chunks/glm.js DELETED
@@ -1 +0,0 @@
1
- import{t as e}from"../vocabulary/glm.js";export{e as default};
package/chunks/gpt.js DELETED
@@ -1 +0,0 @@
1
- import{t as e}from"../vocabulary/gpt.js";export{e as default};
package/chunks/kimi.js DELETED
@@ -1 +0,0 @@
1
- import{t as e}from"../vocabulary/kimi.js";export{e as default};
package/chunks/mimo.js DELETED
@@ -1 +0,0 @@
1
- import{t as e}from"../vocabulary/mimo.js";export{e as default};
package/chunks/minimax.js DELETED
@@ -1 +0,0 @@
1
- import{t as e}from"../vocabulary/minimax.js";export{e as default};
package/chunks/qwen.js DELETED
@@ -1 +0,0 @@
1
- import{t as e}from"../vocabulary/qwen.js";export{e as default};
package/chunks/sdxl.js DELETED
@@ -1 +0,0 @@
1
- import{t as e}from"../vocabulary/sdxl.js";export{e as default};
package/lib/base85Decode.d.ts DELETED
@@ -1,2 +0,0 @@
1
- export declare const decodeBase85: (value: string) => Uint8Array<ArrayBuffer>;
2
- export default decodeBase85;
package/lib/modelAssets.d.ts DELETED
@@ -1,6 +0,0 @@
1
- export type EncodedModelAssetFiles = Record<string, string>;
2
- export type ModelAssetFileContent = Uint8Array | string;
3
- export type ModelAssetFiles = Record<string, ModelAssetFileContent>;
4
- export declare const isCompressedMsgpackFile: (fileName: string) => boolean;
5
- export declare const normalizeModelAssetFileName: (fileName: string) => string;
6
- export declare const prepareEncodedModelAssets: (files: EncodedModelAssetFiles) => Promise<ModelAssetFiles>;