zencommit 0.1.0 → 0.1.2

package/package.json

@@ -1,13 +1,17 @@
 {
   "name": "zencommit",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "module": "./src/index.ts",
   "type": "module",
   "private": false,
   "bin": {
     "zencommit": "bin/zencommit.js"
   },
-  "files": ["src", "bin", "scripts"],
+  "files": [
+    "src",
+    "bin",
+    "scripts"
+  ],
   "scripts": {
     "lint": "eslint .",
     "lint:fix": "eslint . --fix",
@@ -24,6 +28,10 @@
     "access": "public",
     "registry": "https://registry.npmjs.org"
   },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/mboisvertdupras/zencommit"
+  },
   "dependencies": {
     "@ai-sdk/amazon-bedrock": "^4.0.39",
     "@ai-sdk/anthropic": "^3.0.30",
@@ -47,6 +55,7 @@
     "@gitlab/gitlab-ai-provider": "^3.3.1",
     "@openrouter/ai-sdk-provider": "^2.1.1",
     "ai": "^6.0.61",
+    "js-tiktoken": "^1.0.12",
     "yargs": "^18.0.0",
     "yocto-spinner": "^1.0.0"
   },

package/src/commands/default.ts

@@ -6,7 +6,12 @@ import { getRepoRoot } from '../git/repo.js';
 import { getDiff, getFileList, getFileSummary } from '../git/diff.js';
 import { commitMessage } from '../git/commit.js';
 import { buildPrompt, buildPromptWithoutDiff } from '../llm/prompt.js';
-import { computeTokenBudget, countTokens, getEncodingForModel } from '../llm/tokens.js';
+import {
+  computeTokenBudget,
+  countTokens,
+  freeEncoding,
+  getEncodingForModel,
+} from '../llm/tokens.js';
 import { truncateDiffByFile, truncateDiffSmart } from '../llm/truncate.js';
 import { generateCommitMessage } from '../llm/generate.js';
 import { confirmCommit } from '../ui/prompts.js';
@@ -221,7 +226,7 @@ export const runDefaultCommand = async (args: DefaultCommandArgs): Promise<void>
       }
     }
 
-    encoding.free();
+    freeEncoding(encoding);
 
     const prompt = await buildPrompt({ ...promptInput, diffText: truncatedText });
     if (getVerbosity() >= 3) {

package/src/llm/tokens.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from 'vitest';
-import { computeTokenBudget, countTokens, getEncodingForModel } from './tokens.js';
+import { computeTokenBudget, countTokens, freeEncoding, getEncodingForModel } from './tokens.js';
 
 const limits = { context: 100, input: 80, output: 40 };
 
@@ -16,7 +16,7 @@ describe('countTokens', () => {
   it('counts tokens for simple text', () => {
     const encoding = getEncodingForModel('openai/gpt-4o');
     const tokens = countTokens('hello world', encoding);
-    encoding.free();
+    freeEncoding(encoding);
     expect(tokens).toBeGreaterThan(0);
   });
 });

package/src/llm/tokens.ts

@@ -1,4 +1,4 @@
-import { encoding_for_model, get_encoding, type Tiktoken } from '@dqbd/tiktoken';
+import { encodingForModel, getEncoding } from 'js-tiktoken';
 import type { ModelLimits } from '../metadata/types.js';
 
 export interface TokenBudget {
@@ -10,22 +10,31 @@ export interface TokenBudget {
 
 const DEFAULT_ENCODING = 'cl100k_base';
 
-export const getEncodingForModel = (modelId: string): Tiktoken => {
+export type TokenEncoder = {
+  encode: (text: string) => number[];
+  free?: () => void;
+};
+
+export const getEncodingForModel = (modelId: string): TokenEncoder => {
   const modelName = modelId.includes('/') ? (modelId.split('/')[1] ?? modelId) : modelId;
   try {
-    return encoding_for_model(modelName);
+    return encodingForModel(modelName as Parameters<typeof encodingForModel>[0]);
   } catch {
-    return get_encoding(DEFAULT_ENCODING);
+    return getEncoding(DEFAULT_ENCODING);
   }
 };
 
-export const countTokens = (text: string, encoding: Tiktoken): number => {
+export const countTokens = (text: string, encoding: TokenEncoder): number => {
   if (!text) {
     return 0;
   }
   return encoding.encode(text).length;
 };
 
+export const freeEncoding = (encoding: TokenEncoder): void => {
+  encoding.free?.();
+};
+
 export const computeTokenBudget = (
   limits: ModelLimits,
   maxOutputTokens: number,

package/src/llm/truncate.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from 'vitest';
-import { getEncodingForModel } from './tokens.js';
+import { freeEncoding, getEncodingForModel } from './tokens.js';
 import { truncateDiffByFile, truncateDiffSmart } from './truncate.js';
 import type { DiffConfig } from '../config/types.js';
 
@@ -44,7 +44,7 @@ describe('truncateDiffByFile', () => {
   it('truncates and adds marker when budget is tight', () => {
     const encoding = getEncodingForModel('openai/gpt-4o');
     const result = truncateDiffByFile(sampleDiff, 20, encoding);
-    encoding.free();
+    freeEncoding(encoding);
     expect(result.text).toContain('diff --git');
     expect(result.truncated).toBe(true);
   });
@@ -54,7 +54,7 @@ describe('truncateDiffSmart', () => {
   it('includes file summary', () => {
     const encoding = getEncodingForModel('openai/gpt-4o');
     const result = truncateDiffSmart('M foo.ts (+1 -1)', smartDiff, 120, diffConfig, encoding);
-    encoding.free();
+    freeEncoding(encoding);
     expect(result.text).toContain('File summary');
   });
 });

package/src/llm/truncate.ts

@@ -1,6 +1,5 @@
-import type { Tiktoken } from '@dqbd/tiktoken';
 import type { DiffConfig } from '../config/types.js';
-import { countTokens } from './tokens.js';
+import { countTokens, type TokenEncoder } from './tokens.js';
 
 export type TruncateMode = 'full' | 'byFile' | 'smart' | 'summaryOnly';
 
@@ -117,14 +116,14 @@ const parseDiff = (diffText: string): DiffFile[] => {
   return files;
 };
 
-const tokensForLines = (lines: string[], encoding: Tiktoken): number => {
+const tokensForLines = (lines: string[], encoding: TokenEncoder): number => {
   if (lines.length === 0) {
     return 0;
   }
   return countTokens(`${lines.join('\n')}\n`, encoding);
 };
 
-const truncateLinesToBudget = (lines: string[], budget: number, encoding: Tiktoken): string[] => {
+const truncateLinesToBudget = (lines: string[], budget: number, encoding: TokenEncoder): string[] => {
   const output: string[] = [];
   let used = 0;
   for (const line of lines) {
@@ -141,7 +140,7 @@ const truncateLinesToBudget = (lines: string[], budget: number, encoding: Tiktok
 const buildByFileSection = (
   file: DiffFile,
   tokenBudget: number,
-  encoding: Tiktoken,
+  encoding: TokenEncoder,
 ): { lines: string[]; truncated: boolean } => {
   const lines: string[] = [];
   let truncated = false;
@@ -190,7 +189,7 @@ const buildByFileSection = (
   return { lines, truncated };
 };
 
-const getFileTokenSize = (file: DiffFile, encoding: Tiktoken): number => {
+const getFileTokenSize = (file: DiffFile, encoding: TokenEncoder): number => {
   const allLines: string[] = [...file.headerLines];
   for (const hunk of file.hunks) {
     allLines.push(hunk.header, ...hunk.lines.filter((line) => !line.startsWith(' ')));
@@ -201,7 +200,7 @@ const getFileTokenSize = (file: DiffFile, encoding: Tiktoken): number => {
 export const truncateDiffByFile = (
   diffText: string,
   budgetTokens: number,
-  encoding: Tiktoken,
+  encoding: TokenEncoder,
 ): TruncateResult => {
   if (budgetTokens <= 0) {
     return { text: '', usedTokens: 0, truncated: true, mode: 'summaryOnly' };
@@ -459,7 +458,7 @@ export const truncateDiffSmart = (
   diffText: string,
   budgetTokens: number,
   config: DiffConfig,
-  encoding: Tiktoken,
+  encoding: TokenEncoder,
 ): TruncateResult => {
   if (budgetTokens <= 0) {
     return {