repoburg 1.3.12 → 1.3.14

package/backend/packages/tokenpatch/index.spec.ts

@@ -10,10 +10,30 @@ const TS_WASM_PATH = path.join(
   './grammar/tree-sitter-typescript.wasm',
 );
 
-const TSX_WASM_PATH = path.join(__dirname, './grammar/tree-sitter-tsx.wasm');
-
 // Helper to normalize whitespace for robust comparison
-const normalize = (str: string) => str.replace(/\s+/g, ' ').trim();
+const normalize = (str: string) =>
+  str
+    .replace(/}/g, '} ')
+    .replace(/\s+/g, ' ')
+    .trim();
+
+const runTest = async (
+  sourceCode: string,
+  patchCode: string,
+  expectedResult: string,
+) => {
+  const resultTiktoken = await applySnippetPatch(sourceCode, patchCode, {
+    useTiktoken: true,
+  });
+  expect(normalize(resultTiktoken)).toEqual(normalize(expectedResult));
+
+  const resultTreeSitter = await applySnippetPatch(
+    sourceCode,
+    patchCode,
+    TS_WASM_PATH,
+  );
+  expect(normalize(resultTreeSitter)).toEqual(normalize(expectedResult));
+};
 
 describe('applySnippetPatch', () => {
   it('should replace a method body in a class by automatically finding anchor size', async () => {
@@ -39,12 +59,11 @@ class Greeter {
     // A new implementation
     return "Hello, TypeScript!";
   }
-    fc(){ return 42; }
+  fc(){ return 42; }
 
 }
 `;
-    const result = await applySnippetPatch(sourceCode, patchCode, TS_WASM_PATH);
-    expect(normalize(result)).toEqual(normalize(expectedResult));
+    await runTest(sourceCode, patchCode, expectedResult);
   });
 
   it('should replace a data structure definition', async () => {
@@ -67,8 +86,7 @@ interface MyData {
 }
 console.log("hello");
 `;
-    const result = await applySnippetPatch(sourceCode, patchCode, TS_WASM_PATH);
-    expect(normalize(result)).toEqual(normalize(expectedResult));
+    await runTest(sourceCode, patchCode, expectedResult);
   });
 
   it('should replace a full function definition', async () => {
@@ -99,8 +117,7 @@ function calculate() {
 
 export { calculate };
 `;
-    const result = await applySnippetPatch(sourceCode, patchCode, TS_WASM_PATH);
-    expect(normalize(result)).toEqual(normalize(expectedResult));
+    await runTest(sourceCode, patchCode, expectedResult);
   });
 
   it(`should replace a beginning of the file ${SPECIAL_PATCH_BEGIN_FILE_MARKER}`, async () => {
@@ -122,8 +139,7 @@ import { ModuleB } from './moduleB';
 
 console.log('starting up');
 `;
-    const result = await applySnippetPatch(sourceCode, patchCode, TS_WASM_PATH);
-    expect(normalize(result)).toEqual(normalize(expectedResult));
+    await runTest(sourceCode, patchCode, expectedResult);
   });
 
   it(`should handle ${SPECIAL_PATCH_BEGIN_FILE_MARKER} marker with variations`, async () => {
@@ -145,8 +161,7 @@ import { ModuleB } from './moduleB';
 
 console.log('starting up');
 `;
-    const result = await applySnippetPatch(sourceCode, patchCode, TS_WASM_PATH);
-    expect(normalize(result)).toEqual(normalize(expectedResult));
+    await runTest(sourceCode, patchCode, expectedResult);
   });
 
   it(`should replace a end of the file ${SPECIAL_PATCH_END_FILE_MARKER}`, async () => {
@@ -174,8 +189,7 @@ function calculate() {
 export { calculate };
 export { sum };
 `;
-    const result = await applySnippetPatch(sourceCode, patchCode, TS_WASM_PATH);
-    expect(normalize(result)).toEqual(normalize(expectedResult));
+    await runTest(sourceCode, patchCode, expectedResult);
   });
 
   it(`should handle ${SPECIAL_PATCH_END_FILE_MARKER} marker with variations`, async () => {
@@ -203,11 +217,11 @@ function calculate() {
 export { calculate };
 export { sum };
 `;
-    const result = await applySnippetPatch(sourceCode, patchCode, TS_WASM_PATH);
-    expect(normalize(result)).toEqual(normalize(expectedResult));
+    await runTest(sourceCode, patchCode, expectedResult);
   });
   it('should succeed with dynamic anchor sizing when initial anchors are ambiguous', async () => {
     const sourceCode = `
+import dotenv from 'dotenv';
 const config = {
     port: 8080,
     host: 'localhost',
@@ -223,6 +237,7 @@ const config2 = {
 };
 `;
     const patchCode = `
+import dotenv from 'dotenv';
 const config = {
     port: 9000,
     host: 'localhost',
@@ -232,6 +247,7 @@ const config = {
 function connect() {
 `;
     const expectedResult = `
+import dotenv from 'dotenv';
 const config = {
     port: 9000,
     host: 'localhost',
@@ -247,8 +263,7 @@ const config2 = {
     host: 'remote',
 };
 `;
-    const result = await applySnippetPatch(sourceCode, patchCode, TS_WASM_PATH);
-    expect(normalize(result)).toEqual(normalize(expectedResult));
+    await runTest(sourceCode, patchCode, expectedResult);
   });
 
   it('should handle partial type definitions by trimming ambiguous tokens', async () => {
@@ -279,6 +294,8 @@ const config2 = {
      */
     combinedStatusNew2?: everest_appserver_primitive_Text | null;
     /**
+     * Contains Billable Expense line
+     */
 `;
     const expectedResult = `
     /**
@@ -300,8 +317,7 @@ const config2 = {
      */
     containsPrepaidItem?: everest_appserver_primitive_TrueFalse | null;
 `;
-    const result = await applySnippetPatch(sourceCode, patchCode, TS_WASM_PATH);
-    expect(normalize(result)).toEqual(normalize(expectedResult));
+    await runTest(sourceCode, patchCode, expectedResult);
   });
 
   it('should patch TSX correctly by trimming tokens from invalid partial snippets', async () => {
@@ -412,7 +428,10 @@ export function TransactionsPage() {
               <DataTable
                 columns={billColumns}
                 data={bills}
-                onRowClick={(row) => selectMatchable(row.id)}
+                onRowClick={(row) => {
+                  console.log(\`[UI] Selected matchable bill ID\`)
+                  selectMatchable(row.id)
+                }}
                 selectedId={selectedMatchableId}
               />
             </div>
@@ -573,11 +592,6 @@ export function TransactionsPage() {
   )
 }
 `;
-    const result = await applySnippetPatch(
-      sourceCode,
-      patchCode,
-      TSX_WASM_PATH,
-    );
-    expect(normalize(result)).toEqual(normalize(expectedResult));
+    await runTest(sourceCode, patchCode, expectedResult);
   });
-});
+});

package/backend/packages/tokenpatch/index.ts

@@ -1,5 +1,4 @@
 import { initializeParser } from './parser';
-import { collectTokens } from './tokens';
 import {
   handleBeginOfFilePatch,
   handleEndOfFilePatch,
@@ -9,19 +8,41 @@ import {
   SPECIAL_PATCH_BEGIN_FILE_MARKER,
   SPECIAL_PATCH_END_FILE_MARKER,
 } from '../../src/llm-orchestration/parser/parsing.constants';
+import { TokenizerStrategy } from './tokenizer.interface';
+import { TreeSitterTokenizer } from './strategies/tree-sitter-tokenizer';
+import { TiktokenTokenizer } from './strategies/tiktoken-tokenizer';
+
+export interface ApplyPatchOptions {
+  grammarPath?: string;
+  useTiktoken?: boolean;
+}
 
 export async function applySnippetPatch(
   sourceCode: string,
   patchCode: string,
-  grammarPath: string,
+  optionsOrGrammarPath: string | ApplyPatchOptions,
 ): Promise<string> {
-  const parser = await initializeParser(grammarPath);
+  let strategy: TokenizerStrategy;
+  let options: ApplyPatchOptions;
 
-  const sourceTree = parser.parse(sourceCode);
-  if (!sourceTree) {
-    throw new Error('Failed to parse source code.');
+  // Backward compatibility for when the 3rd argument was just grammarPath string
+  if (typeof optionsOrGrammarPath === 'string') {
+    options = { grammarPath: optionsOrGrammarPath };
+  } else {
+    options = optionsOrGrammarPath;
+  }
+
+  if (options.useTiktoken) {
+    strategy = new TiktokenTokenizer();
+  } else if (options.grammarPath) {
+    const parser = await initializeParser(options.grammarPath);
+    strategy = new TreeSitterTokenizer(parser);
+  } else {
+    // Default to Tiktoken if no grammar path is provided
+    strategy = new TiktokenTokenizer();
   }
-  const sourceTokens = collectTokens(sourceTree, sourceCode);
+
+  const sourceTokens = strategy.tokenize(sourceCode);
 
   let patchResult: {
     replaceStart: number;
@@ -34,43 +55,44 @@ export async function applySnippetPatch(
   const beginOfFileRegex = new RegExp(
     `//\\s*${SPECIAL_PATCH_BEGIN_FILE_MARKER}.*`,
   );
-  const endOfFileRegex = new RegExp(
-    `//\\s*${SPECIAL_PATCH_END_FILE_MARKER}.*`,
-  );
+  const endOfFileRegex = new RegExp(`//\\s*${SPECIAL_PATCH_END_FILE_MARKER}.*`);
 
   const hasBeginOfFile = beginOfFileRegex.test(patchCode);
   const hasEndOfFile = endOfFileRegex.test(patchCode);
 
   if (hasBeginOfFile) {
     processedPatchCode = patchCode.replace(beginOfFileRegex, '');
-    const patchTree = parser.parse(processedPatchCode);
-    if (!patchTree) {
-      throw new Error('Failed to parse patch code.');
-    }
-    const patchTokens = collectTokens(patchTree, processedPatchCode).filter(
-      (t) => t.text !== '',
-    );
+    // We need to tokenize the processed patch code
+    // We filter out empty text tokens if any strategy produces them, though likely not needed for tiktoken
+    const patchTokens = strategy
+      .tokenize(processedPatchCode)
+      .filter((t) => t.text !== '' && t.text !== '\n');
     patchResult = handleBeginOfFilePatch(sourceTokens, patchTokens);
   } else if (hasEndOfFile) {
     processedPatchCode = patchCode.replace(endOfFileRegex, '');
-    const patchTree = parser.parse(processedPatchCode);
-    if (!patchTree) {
-      throw new Error('Failed to parse patch code.');
-    }
-    const patchTokens = collectTokens(patchTree, processedPatchCode).filter(
-      (t) => t.text !== '',
-    );
+    const patchTokens = strategy
+      .tokenize(processedPatchCode)
+      .filter((t) => t.text !== '' && t.text !== '\n');
     patchResult = handleEndOfFilePatch(sourceTokens, patchTokens, sourceCode);
   } else {
     processedPatchCode = patchCode.trim();
-    const patchTree = parser.parse(processedPatchCode);
-    if (!patchTree) {
-      throw new Error('Failed to parse patch code.');
+    const patchTokens = strategy
+      .tokenize(processedPatchCode)
+      .filter((t) => t.text !== '');
+
+    // Heuristic: If using Tiktoken, we discard the first and last 3 tokens to improve matching resilience.
+    // This prevents issues where the LLM hallucinating extra delimiters or context at the edges of the snippet.
+    const isTiktoken = options.useTiktoken || !options.grammarPath;
+    if (
+      isTiktoken &&
+      patchTokens.length >= 6 &&
+      patchTokens.length < sourceTokens.length
+    ) {
+      const innerTokens = patchTokens.slice(2, -2);
+      patchResult = handleStandardPatch(sourceTokens, innerTokens);
+    } else {
+      patchResult = handleStandardPatch(sourceTokens, patchTokens);
     }
-    const patchTokens = collectTokens(patchTree, processedPatchCode).filter(
-      (t) => t.text !== '',
-    );
-    patchResult = handleStandardPatch(sourceTokens, patchTokens);
   }
 
   // NOTE: replaceStart/End are byte offsets.
@@ -85,4 +107,4 @@ export async function applySnippetPatch(
   );
 
   return prefix + finalPatchContent + suffix;
-}
+}

package/backend/packages/tokenpatch/patcher.ts

@@ -17,6 +17,23 @@ interface SimplePatchResult {
   replaceEnd: number;
 }
 
+// Helper to prioritize "Ambiguous" errors over "Not found" errors
+const updateLastError = (
+  currentError: Error | null,
+  newError: Error,
+): Error => {
+  if (!currentError) return newError;
+  // If we already have an Ambiguous error, keep it unless the new one is also Ambiguous
+  // (Assuming Ambiguous is more useful/specific than "Not found")
+  const currentIsAmbiguous = currentError.message.includes('Ambiguous');
+  const newIsAmbiguous = newError.message.includes('Ambiguous');
+
+  if (currentIsAmbiguous && !newIsAmbiguous) {
+    return currentError;
+  }
+  return newError;
+};
+
 // Internal helper for the original matching logic
 function _findBeginOfFilePatchLocation(
   sourceTokens: Token[],
@@ -40,7 +57,10 @@ function _findBeginOfFilePatchLocation(
     if (indices.length > 1) {
       const formattedAnchor = formatAnchor(suffixAnchor);
       const locations = indices
-        .map((i) => `line ${sourceTokens[i].startPosition.row + 1}`)
+        .map(
+          (i) =>
+            `line ${sourceTokens[i].startPosition?.row ? sourceTokens[i].startPosition.row + 1 : '?'}`,
+        )
         .join(', ');
       lastError = `Ambiguous suffix anchor. The sequence "${formattedAnchor}" was found at ${indices.length} locations: ${locations}.`;
     }
@@ -83,7 +103,7 @@ export function handleBeginOfFilePatch(
       const patchInsertEnd = patchAttempt[patchAttempt.length - 1].endIndex;
       return { replaceStart, replaceEnd, patchInsertStart, patchInsertEnd };
     } catch (e) {
-      lastError = e as Error;
+      lastError = updateLastError(lastError, e as Error);
       patchAttempt = patchAttempt.slice(1); // Trim one token from the beginning
     }
   }
@@ -116,7 +136,10 @@ function _findEndOfFilePatchLocation(
     if (indices.length > 1) {
       const formattedAnchor = formatAnchor(prefixAnchor);
       const locations = indices
-        .map((i) => `line ${sourceTokens[i].startPosition.row + 1}`)
+        .map(
+          (i) =>
+            `line ${sourceTokens[i].startPosition?.row ? sourceTokens[i].startPosition.row + 1 : '?'}`,
+        )
         .join(', ');
       lastError = `Ambiguous prefix anchor. The sequence "${formattedAnchor}" was found at ${indices.length} locations: ${locations}.`;
     }
@@ -160,7 +183,7 @@ export function handleEndOfFilePatch(
       const patchInsertEnd = patchAttempt[patchAttempt.length - 1].endIndex;
       return { replaceStart, replaceEnd, patchInsertStart, patchInsertEnd };
     } catch (e) {
-      lastError = e as Error;
+      lastError = updateLastError(lastError, e as Error);
       patchAttempt = patchAttempt.slice(0, -1); // Trim one token from the end
     }
   }
@@ -170,12 +193,16 @@ export function handleEndOfFilePatch(
   );
 }
 
-// Internal helper for the original matching logic
-function _findStandardPatchLocation(
+interface PrefixResult {
+  prefixAnchor: Token[];
+  prefixIndex: number;
+  replaceStart: number;
+}
+
+function _findPrefixLocation(
   sourceTokens: Token[],
   patchTokens: Token[],
-): SimplePatchResult {
-  // 1. Find smallest unique prefix
+): PrefixResult {
   let prefixAnchor: Token[] | null = null;
   let prefixIndex: number | null = null;
   let bestPrefixError: string | null = null;
@@ -193,7 +220,10 @@ function _findStandardPatchLocation(
     if (prefixIndices.length > 1) {
       const formatted = formatAnchor(currentPrefix);
       const locations = prefixIndices
-        .map((i) => `line ${sourceTokens[i].startPosition.row + 1}`)
+        .map(
+          (i) =>
+            `line ${sourceTokens[i].startPosition?.row ? sourceTokens[i].startPosition.row + 1 : '?'}`,
+        )
         .join(', ');
       bestPrefixError = `Ambiguous prefix anchor. The sequence "${formatted}" was found at ${prefixIndices.length} locations: ${locations}.`;
     }
@@ -207,12 +237,24 @@ function _findStandardPatchLocation(
   }
 
   if (!prefixAnchor || prefixIndex === null) {
-    throw new Error(
-      bestPrefixError || 'Could not find a unique prefix anchor.',
-    );
+    throw new Error(bestPrefixError || 'Could not find a unique prefix anchor.');
   }
 
-  // 2. Find smallest unique suffix after prefix
+  const replaceStart = sourceTokens[prefixIndex].startIndex;
+
+  return { prefixAnchor, prefixIndex, replaceStart };
+}
+
+interface SuffixResult {
+  replaceEnd: number;
+}
+
+function _findSuffixLocation(
+  sourceTokens: Token[],
+  patchTokens: Token[],
+  prefixAnchor: Token[],
+  prefixIndex: number,
+): SuffixResult {
   let suffixAnchor: Token[] | null = null;
   let suffixIndex: number | null = null;
   let bestSuffixError: string | null = null;
@@ -239,7 +281,9 @@ function _findStandardPatchLocation(
         .map(
           (i) =>
             `line ${
-              sourceTokens[searchStartIndex + i].startPosition.row + 1
+              sourceTokens[searchStartIndex + i].startPosition?.row
+                ? sourceTokens[searchStartIndex + i].startPosition.row + 1
+                : '?'
             }`,
         )
         .join(', ');
@@ -251,7 +295,9 @@ function _findStandardPatchLocation(
       throw new Error(bestSuffixError);
     }
     const prefixLocation = `line ${
-      sourceTokens[prefixIndex].startPosition.row + 1
+      sourceTokens[prefixIndex].startPosition?.row
+        ? sourceTokens[prefixIndex].startPosition.row + 1
+        : '?'
     }`;
     const formattedPrefix = formatAnchor(prefixAnchor);
     const smallestSuffix = formatAnchor(
@@ -264,12 +310,10 @@ function _findStandardPatchLocation(
     );
   }
 
-  // 3. Apply patch
-  const replaceStart = sourceTokens[prefixIndex].startIndex;
   const replaceEnd =
     sourceTokens[suffixIndex + suffixAnchor.length - 1].endIndex;
 
-  return { replaceStart, replaceEnd };
+  return { replaceEnd };
 }
 
 export function handleStandardPatch(
@@ -282,27 +326,64 @@ export function handleStandardPatch(
     );
   }
 
-  let patchAttempt = [...originalPatchTokens];
+  let startTrim = 0;
+  const endTrim = 0;
   let lastError: Error | null = null;
 
-  // Outer loop for trimming tokens from both ends
+  let prefixInfo: PrefixResult | null = null;
+
+  // Loop 1: Find Prefix (trim from start)
+  let patchAttempt = [...originalPatchTokens];
   while (patchAttempt.length >= 2) {
     try {
-      const { replaceStart, replaceEnd } = _findStandardPatchLocation(
+      prefixInfo = _findPrefixLocation(sourceTokens, patchAttempt);
+      // Success finding prefix
+      startTrim = originalPatchTokens.length - patchAttempt.length;
+      break;
+    } catch (e) {
+      lastError = updateLastError(lastError, e as Error);
+      // Trim one token from the start
+      patchAttempt = patchAttempt.slice(1);
+    }
+  }
+
+  if (!prefixInfo) {
+    throw new Error(
+      `Failed to apply patch. Could not find a unique prefix anchor, even after trimming tokens. Last known error: ${lastError?.message}`,
+    );
+  }
+
+  // Loop 2: Find Suffix (trim from end)
+  // Reset patchAttempt to start from the found prefix startTrim, but allow trimming end
+  patchAttempt = originalPatchTokens.slice(startTrim);
+  lastError = null; // Reset last error for suffix search phase
+
+  while (patchAttempt.length >= prefixInfo.prefixAnchor.length + 1) {
+    // Need at least prefix + 1 token? Or just prefix + suffix?
+    try {
+      const { replaceEnd } = _findSuffixLocation(
         sourceTokens,
         patchAttempt,
+        prefixInfo.prefixAnchor,
+        prefixInfo.prefixIndex,
       );
+      // Success finding suffix
       const patchInsertStart = patchAttempt[0].startIndex;
       const patchInsertEnd = patchAttempt[patchAttempt.length - 1].endIndex;
-      return { replaceStart, replaceEnd, patchInsertStart, patchInsertEnd };
+      return {
+        replaceStart: prefixInfo.replaceStart,
+        replaceEnd,
+        patchInsertStart,
+        patchInsertEnd,
+      };
     } catch (e) {
-      lastError = e as Error;
-      // Trim one token from the start and one from the end for the next attempt
-      patchAttempt = patchAttempt.slice(1, -1);
+      lastError = updateLastError(lastError, e as Error);
+      // Trim one token from the end
+      patchAttempt = patchAttempt.slice(0, -1);
     }
   }
 
   throw new Error(
-    `Failed to apply patch. Could not find a unique anchor in the source file, even after trimming ambiguous tokens. Last known error: ${lastError?.message}`,
+    `Failed to apply patch. Could not find a unique suffix anchor, even after trimming tokens. Last known error: ${lastError?.message}`,
   );
 }

package/backend/packages/tokenpatch/strategies/tiktoken-tokenizer.ts

@@ -0,0 +1,35 @@
+import { getEncoding } from 'js-tiktoken';
+import { Token } from '../types';
+import { TokenizerStrategy } from '../tokenizer.interface';
+
+export class TiktokenTokenizer implements TokenizerStrategy {
+  // Use cl100k_base (GPT-4) as the standard encoding
+  private enc = getEncoding('cl100k_base');
+
+  tokenize(content: string): Token[] {
+    const tokens: Token[] = [];
+    const encoded = this.enc.encode(content);
+    
+    let currentIndex = 0;
+    
+    // Iterate through token IDs, decode them individually to get text and length.
+    // This allows us to reconstruct the offsets (startIndex/endIndex).
+    for (const tokenId of encoded) {
+        // decoding a single token is the only way to get its exact text representation
+        // to map back to the source string indices.
+        const text = this.enc.decode([tokenId]);
+        const length = text.length;
+        
+        tokens.push({
+            text,
+            type: 'bpe',
+            startIndex: currentIndex,
+            endIndex: currentIndex + length,
+            // startPosition is not calculated for Tiktoken strategy as it's computationally expensive 
+            // and not strictly required for the patching algorithm which relies on text matching.
+        });
+        currentIndex += length;
+    }
+    return tokens;
+  }
+}

package/backend/packages/tokenpatch/strategies/tree-sitter-tokenizer.ts

@@ -0,0 +1,37 @@
+import { Token } from '../types';
+import { TokenizerStrategy } from '../tokenizer.interface';
+import type { Tree, Node, Parser } from 'web-tree-sitter';
+
+export class TreeSitterTokenizer implements TokenizerStrategy {
+  constructor(private parser: Parser) {}
+
+  tokenize(content: string): Token[] {
+    const tree = this.parser.parse(content);
+    return this.collectTokens(tree, content);
+  }
+
+  private collectTokens(tree: Tree, code: string): Token[] {
+    const tokens: Token[] = [];
+
+    function visit(node: Node) {
+      if (node.childCount === 0) {
+        tokens.push({
+          text: code.slice(node.startIndex, node.endIndex),
+          type: node.type,
+          startIndex: node.startIndex,
+          endIndex: node.endIndex,
+          startPosition: node.startPosition,
+        });
+        return;
+      }
+      // By iterating over all children (not just named), we include punctuation
+      for (let i = 0; i < node.childCount; i++) {
+        const child = node.child(i);
+        if (child) visit(child);
+      }
+    }
+
+    visit(tree.rootNode);
+    return tokens;
+  }
+}

package/backend/packages/tokenpatch/tokenizer.interface.ts

@@ -0,0 +1,5 @@
+import { Token } from './types';
+
+export interface TokenizerStrategy {
+  tokenize(content: string): Token[];
+}

package/backend/packages/tokenpatch/tokens.ts

@@ -1,33 +1,15 @@
-import type { Tree, Node } from 'web-tree-sitter';
 import type { Token } from './types';
 
-export function collectTokens(tree: Tree, code: string): Token[] {
-  const tokens: Token[] = [];
-
-  function visit(node: Node) {
-    if (node.childCount === 0) {
-      tokens.push({
-        text: code.slice(node.startIndex, node.endIndex),
-        type: node.type,
-        startIndex: node.startIndex,
-        endIndex: node.endIndex,
-        startPosition: node.startPosition,
-      });
-      return;
-    }
-    // By iterating over all children (not just named), we include punctuation
-    // which is critical for anchor-based matching.
-    for (let i = 0; i < node.childCount; i++) {
-      const child = node.child(i);
-      if (child) visit(child);
-    }
-  }
-
-  visit(tree.rootNode);
-  return tokens;
-}
-
 export function tokensEqual(a: Token, b: Token): boolean {
+  // If using Tiktoken (BPE), we relax the comparison to ignore whitespace differences
+  // and newlines by trimming and stripping them. This helps when the patch might
+  // have slightly different indentation or spacing than the source.
+  if (a.type === 'bpe' || b.type === 'bpe') {
+    return (
+      a.text.replace(/\r?\n/g, '').trim() ===
+      b.text.replace(/\r?\n/g, '').trim()
+    );
+  }
   return a.text === b.text;
 }
 
@@ -47,4 +29,4 @@ export function findAllSequences(haystack: Token[], needle: Token[]): number[] {
     indices.push(i);
   }
   return indices;
-}
+}

package/backend/packages/tokenpatch/types.ts

@@ -2,8 +2,8 @@ import type { Point } from 'web-tree-sitter';
 
 export interface Token {
   text: string;
-  type: string;
-  startIndex: number; // byte offsets (tree-sitter)
+  type?: string;
+  startIndex: number; // byte offsets
   endIndex: number;
-  startPosition: Point;
-}
+  startPosition?: Point;
+}