repoburg 1.3.11 → 1.3.13

package/backend/dist/packages/tokenpatch/index.d.ts

@@ -1 +1,5 @@
-export declare function applySnippetPatch(sourceCode: string, patchCode: string, grammarPath: string): Promise<string>;
+export interface ApplyPatchOptions {
+    grammarPath?: string;
+    useTiktoken?: boolean;
+}
+export declare function applySnippetPatch(sourceCode: string, patchCode: string, optionsOrGrammarPath: string | ApplyPatchOptions): Promise<string>;

package/backend/dist/packages/tokenpatch/index.js

@@ -2,47 +2,65 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.applySnippetPatch = applySnippetPatch;
 const parser_1 = require("./parser");
-const tokens_1 = require("./tokens");
 const patcher_1 = require("./patcher");
-async function applySnippetPatch(sourceCode, patchCode, grammarPath) {
-    const parser = await (0, parser_1.initializeParser)(grammarPath);
-    const sourceTree = parser.parse(sourceCode);
-    if (!sourceTree) {
-        throw new Error('Failed to parse source code.');
+const parsing_constants_1 = require("../../src/llm-orchestration/parser/parsing.constants");
+const tree_sitter_tokenizer_1 = require("./strategies/tree-sitter-tokenizer");
+const tiktoken_tokenizer_1 = require("./strategies/tiktoken-tokenizer");
+async function applySnippetPatch(sourceCode, patchCode, optionsOrGrammarPath) {
+    let strategy;
+    let options;
+    if (typeof optionsOrGrammarPath === 'string') {
+        options = { grammarPath: optionsOrGrammarPath };
     }
-    const sourceTokens = (0, tokens_1.collectTokens)(sourceTree, sourceCode);
+    else {
+        options = optionsOrGrammarPath;
+    }
+    if (options.useTiktoken) {
+        strategy = new tiktoken_tokenizer_1.TiktokenTokenizer();
+    }
+    else if (options.grammarPath) {
+        const parser = await (0, parser_1.initializeParser)(options.grammarPath);
+        strategy = new tree_sitter_tokenizer_1.TreeSitterTokenizer(parser);
+    }
+    else {
+        strategy = new tiktoken_tokenizer_1.TiktokenTokenizer();
+    }
+    const sourceTokens = strategy.tokenize(sourceCode);
     let patchResult;
     let processedPatchCode = patchCode;
-    const beginOfFileRegex = /\/\/\s*@begin-of-file.*/;
-    const endOfFileRegex = /\/\/\s*@end-of-file.*/;
+    const beginOfFileRegex = new RegExp(`//\\s*${parsing_constants_1.SPECIAL_PATCH_BEGIN_FILE_MARKER}.*`);
+    const endOfFileRegex = new RegExp(`//\\s*${parsing_constants_1.SPECIAL_PATCH_END_FILE_MARKER}.*`);
     const hasBeginOfFile = beginOfFileRegex.test(patchCode);
     const hasEndOfFile = endOfFileRegex.test(patchCode);
     if (hasBeginOfFile) {
         processedPatchCode = patchCode.replace(beginOfFileRegex, '');
-        const patchTree = parser.parse(processedPatchCode);
-        if (!patchTree) {
-            throw new Error('Failed to parse patch code.');
-        }
-        const patchTokens = (0, tokens_1.collectTokens)(patchTree, processedPatchCode).filter((t) => t.text !== '');
+        const patchTokens = strategy
+            .tokenize(processedPatchCode)
+            .filter((t) => t.text !== '' && t.text !== '\n');
         patchResult = (0, patcher_1.handleBeginOfFilePatch)(sourceTokens, patchTokens);
     }
     else if (hasEndOfFile) {
         processedPatchCode = patchCode.replace(endOfFileRegex, '');
-        const patchTree = parser.parse(processedPatchCode);
-        if (!patchTree) {
-            throw new Error('Failed to parse patch code.');
-        }
-        const patchTokens = (0, tokens_1.collectTokens)(patchTree, processedPatchCode).filter((t) => t.text !== '');
+        const patchTokens = strategy
+            .tokenize(processedPatchCode)
+            .filter((t) => t.text !== '' && t.text !== '\n');
         patchResult = (0, patcher_1.handleEndOfFilePatch)(sourceTokens, patchTokens, sourceCode);
     }
     else {
         processedPatchCode = patchCode.trim();
-        const patchTree = parser.parse(processedPatchCode);
-        if (!patchTree) {
-            throw new Error('Failed to parse patch code.');
+        const patchTokens = strategy
+            .tokenize(processedPatchCode)
+            .filter((t) => t.text !== '');
+        const isTiktoken = options.useTiktoken || !options.grammarPath;
+        if (isTiktoken &&
+            patchTokens.length >= 6 &&
+            patchTokens.length < sourceTokens.length) {
+            const innerTokens = patchTokens.slice(2, -2);
+            patchResult = (0, patcher_1.handleStandardPatch)(sourceTokens, innerTokens);
+        }
+        else {
+            patchResult = (0, patcher_1.handleStandardPatch)(sourceTokens, patchTokens);
         }
-        const patchTokens = (0, tokens_1.collectTokens)(patchTree, processedPatchCode).filter((t) => t.text !== '');
-        patchResult = (0, patcher_1.handleStandardPatch)(sourceTokens, patchTokens);
     }
     const prefix = sourceCode.slice(0, patchResult.replaceStart);
     const suffix = sourceCode.slice(patchResult.replaceEnd);

package/backend/dist/packages/tokenpatch/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../packages/tokenpatch/index.ts"],"names":[],"mappings":";;AAQA,8CAuEC;AA/ED,qCAA4C;AAC5C,qCAAyC;AACzC,uCAImB;AAEZ,KAAK,UAAU,iBAAiB,CACrC,UAAkB,EAClB,SAAiB,EACjB,WAAmB;IAEnB,MAAM,MAAM,GAAG,MAAM,IAAA,yBAAgB,EAAC,WAAW,CAAC,CAAC;IAEnD,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAC5C,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;IAClD,CAAC;IACD,MAAM,YAAY,GAAG,IAAA,sBAAa,EAAC,UAAU,EAAE,UAAU,CAAC,CAAC;IAE3D,IAAI,WAKH,CAAC;IACF,IAAI,kBAAkB,GAAG,SAAS,CAAC;IAEnC,MAAM,gBAAgB,GAAG,yBAAyB,CAAC;IACnD,MAAM,cAAc,GAAG,uBAAuB,CAAC;IAE/C,MAAM,cAAc,GAAG,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACxD,MAAM,YAAY,GAAG,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAEpD,IAAI,cAAc,EAAE,CAAC;QACnB,kBAAkB,GAAG,SAAS,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;QAC7D,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACnD,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACjD,CAAC;QACD,MAAM,WAAW,GAAG,IAAA,sBAAa,EAAC,SAAS,EAAE,kBAAkB,CAAC,CAAC,MAAM,CACrE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,EAAE,CACrB,CAAC;QACF,WAAW,GAAG,IAAA,gCAAsB,EAAC,YAAY,EAAE,WAAW,CAAC,CAAC;IAClE,CAAC;SAAM,IAAI,YAAY,EAAE,CAAC;QACxB,kBAAkB,GAAG,SAAS,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QAC3D,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACnD,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACjD,CAAC;QACD,MAAM,WAAW,GAAG,IAAA,sBAAa,EAAC,SAAS,EAAE,kBAAkB,CAAC,CAAC,MAAM,CACrE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,EAAE,CACrB,CAAC;QACF,WAAW,GAAG,IAAA,8BAAoB,EAAC,YAAY,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC;IAC5E,CAAC;SAAM,CAAC;QACN,kBAAkB,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;QACtC,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACnD,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACjD,CAAC;QACD,MAAM,WAAW,GAAG,IAAA,sBAAa,EAAC,SAAS,EAAE,kBAAkB,CAAC,CAAC,MAAM,CACrE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,EAAE,CACrB,CAAC;QACF,WAAW,GAAG,IAAA,6BAAmB,EAAC,YAAY,EAAE,WAAW,CAAC,CAAC;IAC/D,CAAC;IAMD,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,YAAY,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;IACxD,MAAM,iBAAiB,GAAG,kBAAkB,CAAC,KAAK,CAChD,WAAW,CAAC,gBAAgB,EAC5B,WAAW,CAAC,cAAc,CAC3B,CAAC;IAEF,OAAO,MAAM,GAAG,iBAAiB,GAAG,MAAM,CAAC;AAC7C,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../packages/tokenpatch/index.ts"],"names":[],"mappings":";;AAmBA,8CA0FC;AA7GD,qCAA4C;AAC5C,uCAImB;AACnB,4FAG8D;AAE9D,8EAAyE;AACzE,wEAAoE;AAO7D,KAAK,UAAU,iBAAiB,CACrC,UAAkB,EAClB,SAAiB,EACjB,oBAAgD;IAEhD,IAAI,QAA2B,CAAC;IAChC,IAAI,OAA0B,CAAC;IAG/B,IAAI,OAAO,oBAAoB,KAAK,QAAQ,EAAE,CAAC;QAC7C,OAAO,GAAG,EAAE,WAAW,EAAE,oBAAoB,EAAE,CAAC;IAClD,CAAC;SAAM,CAAC;QACN,OAAO,GAAG,oBAAoB,CAAC;IACjC,CAAC;IAED,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;QACxB,QAAQ,GAAG,IAAI,sCAAiB,EAAE,CAAC;IACrC,CAAC;SAAM,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,MAAM,IAAA,yBAAgB,EAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QAC3D,QAAQ,GAAG,IAAI,2CAAmB,CAAC,MAAM,CAAC,CAAC;IAC7C,CAAC;SAAM,CAAC;QAEN,QAAQ,GAAG,IAAI,sCAAiB,EAAE,CAAC;IACrC,CAAC;IAED,MAAM,YAAY,GAAG,QAAQ,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;IAEnD,IAAI,WAKH,CAAC;IACF,IAAI,kBAAkB,GAAG,SAAS,CAAC;IAEnC,MAAM,gBAAgB,GAAG,IAAI,MAAM,CACjC,SAAS,mDAA+B,IAAI,CAC7C,CAAC;IACF,MAAM,cAAc,GAAG,IAAI,MAAM,CAAC,SAAS,iDAA6B,IAAI,CAAC,CAAC;IAE9E,MAAM,cAAc,GAAG,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACxD,MAAM,YAAY,GAAG,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAEpD,IAAI,cAAc,EAAE,CAAC;QACnB,kBAAkB,GAAG,SAAS,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;QAG7D,MAAM,WAAW,GAAG,QAAQ;aACzB,QAAQ,CAAC,kBAAkB,CAAC;aAC5B,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,EAAE,IAAI,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;QACnD,WAAW,GAAG,IAAA,gCAAsB,EAAC,YAAY,EAAE,WAAW,CAAC,CAAC;IAClE,CAAC;SAAM,IAAI,YAAY,EAAE,CAAC;QACxB,kBAAkB,GAAG,SAAS,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QAC3D,MAAM,WAAW,GAAG,QAAQ;aACzB,QAAQ,CAAC,kBAAkB,CAAC;aAC5B,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,EAAE,IAAI,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;QACnD,WAAW,GAAG,IAAA,8BAAoB,EAAC,YAAY,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC;IAC5E,CAAC;SAAM,CAAC;QACN,kBAAkB,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;QACtC,MAAM,WAAW,GAAG,QAAQ;aACzB,QAAQ,CAAC,kBAAkB,CAAC;aAC5B,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC;QAIhC,MAAM,UAAU,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;QAC/D,IACE,UAAU;YACV,WAAW,CAAC,MAAM,IAAI,CAAC;YACvB,WAAW,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM,EACxC,CAAC;YACD,MAAM,WAAW,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAC7C,WAAW,GAAG,IAAA,6BAAmB,EAAC,YAAY,EAAE,WAAW,CAAC,CAAC;QAC/D,CAAC;aAAM,CAAC;YACN,WAAW,GAAG,IAAA,6BAAmB,EAAC,YAAY,EAAE,WAAW,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;IAMD,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,YAAY,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;IACxD,MAAM,iBAAiB,GAAG,kBAAkB,CAAC,KAAK,CAChD,WAAW,CAAC,gBAAgB,EAC5B,WAAW,CAAC,cAAc,CAC3B,CAAC;IAEF,OAAO,MAAM,GAAG,iBAAiB,GAAG,MAAM,CAAC;AAC7C,CAAC"}

package/backend/dist/packages/tokenpatch/patcher.js

@@ -4,6 +4,17 @@ exports.handleBeginOfFilePatch = handleBeginOfFilePatch;
 exports.handleEndOfFilePatch = handleEndOfFilePatch;
 exports.handleStandardPatch = handleStandardPatch;
 const tokens_1 = require("./tokens");
+const parsing_constants_1 = require("../../src/llm-orchestration/parser/parsing.constants");
+const updateLastError = (currentError, newError) => {
+    if (!currentError)
+        return newError;
+    const currentIsAmbiguous = currentError.message.includes('Ambiguous');
+    const newIsAmbiguous = newError.message.includes('Ambiguous');
+    if (currentIsAmbiguous && !newIsAmbiguous) {
+        return currentError;
+    }
+    return newError;
+};
 function _findBeginOfFilePatchLocation(sourceTokens, patchTokens) {
     let replaceStart = null;
     let replaceEnd = null;
@@ -20,7 +31,7 @@ function _findBeginOfFilePatchLocation(sourceTokens, patchTokens) {
         if (indices.length > 1) {
             const formattedAnchor = (0, tokens_1.formatAnchor)(suffixAnchor);
             const locations = indices
-                .map((i) => `line ${sourceTokens[i].startPosition.row + 1}`)
+                .map((i) => `line ${sourceTokens[i].startPosition?.row ? sourceTokens[i].startPosition.row + 1 : '?'}`)
                 .join(', ');
             lastError = `Ambiguous suffix anchor. The sequence "${formattedAnchor}" was found at ${indices.length} locations: ${locations}.`;
         }
@@ -34,7 +45,7 @@ function _findBeginOfFilePatchLocation(sourceTokens, patchTokens) {
 }
 function handleBeginOfFilePatch(sourceTokens, originalPatchTokens) {
     if (originalPatchTokens.length === 0) {
-        throw new Error('Patch is empty after removing @begin-of-file marker.');
+        throw new Error(`Patch is empty after removing ${parsing_constants_1.SPECIAL_PATCH_BEGIN_FILE_MARKER} marker.`);
     }
     let patchAttempt = [...originalPatchTokens];
     let lastError = null;
@@ -46,11 +57,11 @@ function handleBeginOfFilePatch(sourceTokens, originalPatchTokens) {
             return { replaceStart, replaceEnd, patchInsertStart, patchInsertEnd };
         }
         catch (e) {
-            lastError = e;
+            lastError = updateLastError(lastError, e);
             patchAttempt = patchAttempt.slice(1);
         }
     }
-    throw new Error(`Failed to apply @begin-of-file patch. Could not find a unique anchor, even after trimming tokens. Last known error: ${lastError?.message}`);
+    throw new Error(`Failed to apply ${parsing_constants_1.SPECIAL_PATCH_BEGIN_FILE_MARKER} patch. Could not find a unique anchor, even after trimming tokens. Last known error: ${lastError?.message}`);
 }
 function _findEndOfFilePatchLocation(sourceTokens, patchTokens, sourceCode) {
     let replaceStart = null;
@@ -67,7 +78,7 @@ function _findEndOfFilePatchLocation(sourceTokens, patchTokens, sourceCode) {
         if (indices.length > 1) {
             const formattedAnchor = (0, tokens_1.formatAnchor)(prefixAnchor);
             const locations = indices
-                .map((i) => `line ${sourceTokens[i].startPosition.row + 1}`)
+                .map((i) => `line ${sourceTokens[i].startPosition?.row ? sourceTokens[i].startPosition.row + 1 : '?'}`)
                 .join(', ');
             lastError = `Ambiguous prefix anchor. The sequence "${formattedAnchor}" was found at ${indices.length} locations: ${locations}.`;
         }
@@ -81,7 +92,7 @@ function _findEndOfFilePatchLocation(sourceTokens, patchTokens, sourceCode) {
 }
 function handleEndOfFilePatch(sourceTokens, originalPatchTokens, sourceCode) {
     if (originalPatchTokens.length === 0) {
-        throw new Error('Patch is empty after removing @end-of-file marker.');
+        throw new Error(`Patch is empty after removing ${parsing_constants_1.SPECIAL_PATCH_END_FILE_MARKER} marker.`);
     }
     let patchAttempt = [...originalPatchTokens];
     let lastError = null;
@@ -93,13 +104,13 @@ function handleEndOfFilePatch(sourceTokens, originalPatchTokens, sourceCode) {
             return { replaceStart, replaceEnd, patchInsertStart, patchInsertEnd };
         }
         catch (e) {
-            lastError = e;
+            lastError = updateLastError(lastError, e);
             patchAttempt = patchAttempt.slice(0, -1);
         }
     }
-    throw new Error(`Failed to apply @end-of-file patch. Could not find a unique anchor, even after trimming tokens. Last known error: ${lastError?.message}`);
+    throw new Error(`Failed to apply ${parsing_constants_1.SPECIAL_PATCH_END_FILE_MARKER} patch. Could not find a unique anchor, even after trimming tokens. Last known error: ${lastError?.message}`);
 }
-function _findStandardPatchLocation(sourceTokens, patchTokens) {
+function _findPrefixLocation(sourceTokens, patchTokens) {
     let prefixAnchor = null;
     let prefixIndex = null;
     let bestPrefixError = null;
@@ -115,7 +126,7 @@ function _findStandardPatchLocation(sourceTokens, patchTokens) {
         if (prefixIndices.length > 1) {
             const formatted = (0, tokens_1.formatAnchor)(currentPrefix);
             const locations = prefixIndices
-                .map((i) => `line ${sourceTokens[i].startPosition.row + 1}`)
+                .map((i) => `line ${sourceTokens[i].startPosition?.row ? sourceTokens[i].startPosition.row + 1 : '?'}`)
                 .join(', ');
             bestPrefixError = `Ambiguous prefix anchor. The sequence "${formatted}" was found at ${prefixIndices.length} locations: ${locations}.`;
         }
@@ -130,6 +141,10 @@ function _findStandardPatchLocation(sourceTokens, patchTokens) {
     if (!prefixAnchor || prefixIndex === null) {
         throw new Error(bestPrefixError || 'Could not find a unique prefix anchor.');
     }
+    const replaceStart = sourceTokens[prefixIndex].startIndex;
+    return { prefixAnchor, prefixIndex, replaceStart };
+}
+function _findSuffixLocation(sourceTokens, patchTokens, prefixAnchor, prefixIndex) {
     let suffixAnchor = null;
     let suffixIndex = null;
     let bestSuffixError = null;
@@ -147,7 +162,9 @@ function _findStandardPatchLocation(sourceTokens, patchTokens) {
         if (suffixIndices.length > 1) {
             const formatted = (0, tokens_1.formatAnchor)(currentSuffix);
             const locations = suffixIndices
-                .map((i) => `line ${sourceTokens[searchStartIndex + i].startPosition.row + 1}`)
+                .map((i) => `line ${sourceTokens[searchStartIndex + i].startPosition?.row
+                ? sourceTokens[searchStartIndex + i].startPosition.row + 1
+                : '?'}`)
                 .join(', ');
             bestSuffixError = `Ambiguous suffix anchor. The sequence "${formatted}" was found ${suffixIndices.length} times after the prefix: ${locations}.`;
         }
@@ -156,34 +173,59 @@ function _findStandardPatchLocation(sourceTokens, patchTokens) {
         if (bestSuffixError) {
             throw new Error(bestSuffixError);
         }
-        const prefixLocation = `line ${sourceTokens[prefixIndex].startPosition.row + 1}`;
+        const prefixLocation = `line ${sourceTokens[prefixIndex].startPosition?.row
+            ? sourceTokens[prefixIndex].startPosition.row + 1
+            : '?'}`;
         const formattedPrefix = (0, tokens_1.formatAnchor)(prefixAnchor);
         const smallestSuffix = (0, tokens_1.formatAnchor)(patchTokens.slice(patchTokens.length - 1));
         throw new Error(`Could not find a unique suffix anchor after the prefix anchor "${formattedPrefix}" (found at ${prefixLocation}). ` +
             `The smallest suffix searched for ("${smallestSuffix}") was not found after it.`);
     }
-    const replaceStart = sourceTokens[prefixIndex].startIndex;
     const replaceEnd = sourceTokens[suffixIndex + suffixAnchor.length - 1].endIndex;
-    return { replaceStart, replaceEnd };
+    return { replaceEnd };
 }
 function handleStandardPatch(sourceTokens, originalPatchTokens) {
     if (originalPatchTokens.length < 2) {
         throw new Error('Patch must contain at least two tokens to form a prefix and a suffix.');
     }
-    let patchAttempt = [...originalPatchTokens];
+    let startTrim = 0;
+    const endTrim = 0;
     let lastError = null;
+    let prefixInfo = null;
+    let patchAttempt = [...originalPatchTokens];
     while (patchAttempt.length >= 2) {
         try {
-            const { replaceStart, replaceEnd } = _findStandardPatchLocation(sourceTokens, patchAttempt);
+            prefixInfo = _findPrefixLocation(sourceTokens, patchAttempt);
+            startTrim = originalPatchTokens.length - patchAttempt.length;
+            break;
+        }
+        catch (e) {
+            lastError = updateLastError(lastError, e);
+            patchAttempt = patchAttempt.slice(1);
+        }
+    }
+    if (!prefixInfo) {
+        throw new Error(`Failed to apply patch. Could not find a unique prefix anchor, even after trimming tokens. Last known error: ${lastError?.message}`);
+    }
+    patchAttempt = originalPatchTokens.slice(startTrim);
+    lastError = null;
+    while (patchAttempt.length >= prefixInfo.prefixAnchor.length + 1) {
+        try {
+            const { replaceEnd } = _findSuffixLocation(sourceTokens, patchAttempt, prefixInfo.prefixAnchor, prefixInfo.prefixIndex);
             const patchInsertStart = patchAttempt[0].startIndex;
             const patchInsertEnd = patchAttempt[patchAttempt.length - 1].endIndex;
-            return { replaceStart, replaceEnd, patchInsertStart, patchInsertEnd };
+            return {
+                replaceStart: prefixInfo.replaceStart,
+                replaceEnd,
+                patchInsertStart,
+                patchInsertEnd,
+            };
         }
         catch (e) {
-            lastError = e;
-            patchAttempt = patchAttempt.slice(1, -1);
+            lastError = updateLastError(lastError, e);
+            patchAttempt = patchAttempt.slice(0, -1);
         }
     }
-    throw new Error(`Failed to apply patch. Could not find a unique anchor in the source file, even after trimming ambiguous tokens. Last known error: ${lastError?.message}`);
+    throw new Error(`Failed to apply patch. Could not find a unique suffix anchor, even after trimming tokens. Last known error: ${lastError?.message}`);
 }
 //# sourceMappingURL=patcher.js.map

package/backend/dist/packages/tokenpatch/patcher.js.map

@@ -1 +1 @@
-{"version":3,"file":"patcher.js","sourceRoot":"","sources":["../../../packages/tokenpatch/patcher.ts"],"names":[],"mappings":";;AAyDA,wDA8BC;AA2CD,oDAgCC;AAwGD,kDAiCC;AA1SD,qCAA0D;AAe1D,SAAS,6BAA6B,CACpC,YAAqB,EACrB,WAAoB;IAEpB,IAAI,YAAY,GAAkB,IAAI,CAAC;IACvC,IAAI,UAAU,GAAkB,IAAI,CAAC;IACrC,IAAI,SAAS,GAAG,EAAE,CAAC;IAEnB,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,IAAI,WAAW,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,CAAC;QACxE,MAAM,YAAY,GAAG,WAAW,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,GAAG,UAAU,CAAC,CAAC;QACxE,MAAM,OAAO,GAAG,IAAA,yBAAgB,EAAC,YAAY,EAAE,YAAY,CAAC,CAAC;QAE7D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,YAAY,GAAG,CAAC,CAAC;YACjB,MAAM,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YAC/B,UAAU,GAAG,YAAY,CAAC,WAAW,GAAG,UAAU,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC;YACjE,MAAM;QACR,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,eAAe,GAAG,IAAA,qBAAY,EAAC,YAAY,CAAC,CAAC;YACnD,MAAM,SAAS,GAAG,OAAO;iBACtB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,YAAY,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC;iBAC3D,IAAI,CAAC,IAAI,CAAC,CAAC;YACd,SAAS,GAAG,0CAA0C,eAAe,kBAAkB,OAAO,CAAC,MAAM,eAAe,SAAS,GAAG,CAAC;QACnI,CAAC;IACH,CAAC;IAED,IAAI,YAAY,KAAK,IAAI,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;QACjD,MAAM,cAAc,GAAG,WAAW,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACjE,MAAM,IAAI,KAAK,CACb,SAAS;YACP,0CAA0C,IAAA,qBAAY,EACpD,cAAc,CACf,yBAAyB,CAC7B,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,CAAC;AACtC,CAAC;AAED,SAAgB,sBAAsB,CACpC,YAAqB,EACrB,mBAA4B;IAE5B,IAAI,mBAAmB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;IAC1E,CAAC;IAED,IAAI,YAAY,GAAG,CAAC,GAAG,mBAAmB,CAAC,CAAC;IAC5C,IAAI,SAAS,GAAiB,IAAI,CAAC;IAGnC,OAAO,YAAY,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAChC,IAAI,CAAC;YACH,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,GAAG,6BAA6B,CAChE,YAAY,EACZ,YAAY,CACb,CAAC;YACF,MAAM,gBAAgB,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;YACpD,MAAM,cAAc,GAAG,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC;YACtE,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,gBAAgB,EAAE,cAAc,EAAE,CAAC;QACxE,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,SAAS,GAAG,CAAU,CAAC;YACvB,YAAY,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IAED,MAAM,IAAI,KAAK,CACb,uHAAuH,SAAS,EAAE,OAAO,EAAE,CAC5I,CAAC;AACJ,CAAC;AAGD,SAAS,2BAA2B,CAClC,YAAqB,EACrB,WAAoB,EACpB,UAAkB;IAElB,IAAI,YAAY,GAAkB,IAAI,CAAC;IACvC,IAAI,UAAU,GAAkB,IAAI,CAAC;IACrC,IAAI,SAAS,GAAG,EAAE,CAAC;IAEnB,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,IAAI,WAAW,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,CAAC;QACxE,MAAM,YAAY,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;QACtD,MAAM,OAAO,GAAG,IAAA,yBAAgB,EAAC,YAAY,EAAE,YAAY,CAAC,CAAC;QAE7D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,YAAY,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;YACnD,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC;YAC/B,MAAM;QACR,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,eAAe,GAAG,IAAA,qBAAY,EAAC,YAAY,CAAC,CAAC;YACnD,MAAM,SAAS,GAAG,OAAO;iBACtB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,YAAY,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC;iBAC3D,IAAI,CAAC,IAAI,CAAC,CAAC;YACd,SAAS,GAAG,0CAA0C,eAAe,kBAAkB,OAAO,CAAC,MAAM,eAAe,SAAS,GAAG,CAAC;QACnI,CAAC;IACH,CAAC;IAED,IAAI,YAAY,KAAK,IAAI,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;QACjD,MAAM,cAAc,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC/C,MAAM,IAAI,KAAK,CACb,SAAS;YACP,0CAA0C,IAAA,qBAAY,EACpD,cAAc,CACf,yBAAyB,CAC7B,CAAC;IACJ,CAAC;IACD,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,CAAC;AACtC,CAAC;AAED,SAAgB,oBAAoB,CAClC,YAAqB,EACrB,mBAA4B,EAC5B,UAAkB;IAElB,IAAI,mBAAmB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IACxE,CAAC;IAED,IAAI,YAAY,GAAG,CAAC,GAAG,mBAAmB,CAAC,CAAC;IAC5C,IAAI,SAAS,GAAiB,IAAI,CAAC;IAGnC,OAAO,YAAY,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAChC,IAAI,CAAC;YACH,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,GAAG,2BAA2B,CAC9D,YAAY,EACZ,YAAY,EACZ,UAAU,CACX,CAAC;YACF,MAAM,gBAAgB,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;YACpD,MAAM,cAAc,GAAG,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC;YACtE,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,gBAAgB,EAAE,cAAc,EAAE,CAAC;QACxE,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,SAAS,GAAG,CAAU,CAAC;YACvB,YAAY,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IAED,MAAM,IAAI,KAAK,CACb,qHAAqH,SAAS,EAAE,OAAO,EAAE,CAC1I,CAAC;AACJ,CAAC;AAGD,SAAS,0BAA0B,CACjC,YAAqB,EACrB,WAAoB;IAGpB,IAAI,YAAY,GAAmB,IAAI,CAAC;IACxC,IAAI,WAAW,GAAkB,IAAI,CAAC;IACtC,IAAI,eAAe,GAAkB,IAAI,CAAC;IAE1C,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,GAAG,WAAW,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,CAAC;QACvE,MAAM,aAAa,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;QACvD,MAAM,aAAa,GAAG,IAAA,yBAAgB,EAAC,YAAY,EAAE,aAAa,CAAC,CAAC;QAEpE,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,YAAY,GAAG,aAAa,CAAC;YAC7B,WAAW,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;YAC/B,eAAe,GAAG,IAAI,CAAC;YACvB,MAAM;QACR,CAAC;QACD,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAG,IAAA,qBAAY,EAAC,aAAa,CAAC,CAAC;YAC9C,MAAM,SAAS,GAAG,aAAa;iBAC5B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,YAAY,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC;iBAC3D,IAAI,CAAC,IAAI,CAAC,CAAC;YACd,eAAe,GAAG,0CAA0C,SAAS,kBAAkB,aAAa,CAAC,MAAM,eAAe,SAAS,GAAG,CAAC;QACzI,CAAC;QACD,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,eAAe,EAAE,CAAC;gBACrB,MAAM,SAAS,GAAG,IAAA,qBAAY,EAAC,aAAa,CAAC,CAAC;gBAC9C,eAAe,GAAG,6BAA6B,SAAS,GAAG,CAAC;YAC9D,CAAC;YACD,MAAM;QACR,CAAC;IACH,CAAC;IAED,IAAI,CAAC,YAAY,IAAI,WAAW,KAAK,IAAI,EAAE,CAAC;QAC1C,MAAM,IAAI,KAAK,CACb,eAAe,IAAI,wCAAwC,CAC5D,CAAC;IACJ,CAAC;IAGD,IAAI,YAAY,GAAmB,IAAI,CAAC;IACxC,IAAI,WAAW,GAAkB,IAAI,CAAC;IACtC,IAAI,eAAe,GAAkB,IAAI,CAAC;IAC1C,MAAM,gBAAgB,GAAG,WAAW,GAAG,YAAY,CAAC,MAAM,CAAC;IAC3D,MAAM,iBAAiB,GAAG,YAAY,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;IAE/D,KACE,IAAI,UAAU,GAAG,CAAC,EAClB,UAAU,IAAI,WAAW,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM,EACtD,UAAU,EAAE,EACZ,CAAC;QACD,MAAM,aAAa,GAAG,WAAW,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,GAAG,UAAU,CAAC,CAAC;QACzE,MAAM,aAAa,GAAG,IAAA,yBAAgB,EAAC,iBAAiB,EAAE,aAAa,CAAC,CAAC;QAEzE,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,YAAY,GAAG,aAAa,CAAC;YAC7B,WAAW,GAAG,gBAAgB,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;YAClD,eAAe,GAAG,IAAI,CAAC;YACvB,MAAM;QACR,CAAC;QACD,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAG,IAAA,qBAAY,EAAC,aAAa,CAAC,CAAC;YAC9C,MAAM,SAAS,GAAG,aAAa;iBAC5B,GAAG,CACF,CAAC,CAAC,EAAE,EAAE,CACJ,QACE,YAAY,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC,GAAG,GAAG,CACzD,EAAE,CACL;iBACA,IAAI,CAAC,IAAI,CAAC,CAAC;YACd,eAAe,GAAG,0CAA0C,SAAS,eAAe,aAAa,CAAC,MAAM,4BAA4B,SAAS,GAAG,CAAC;QACnJ,CAAC;IACH,CAAC;IACD,IAAI,CAAC,YAAY,IAAI,WAAW,KAAK,IAAI,EAAE,CAAC;QAC1C,IAAI,eAAe,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC;QACnC,CAAC;QACD,MAAM,cAAc,GAAG,QACrB,YAAY,CAAC,WAAW,CAAC,CAAC,aAAa,CAAC,GAAG,GAAG,CAChD,EAAE,CAAC;QACH,MAAM,eAAe,GAAG,IAAA,qBAAY,EAAC,YAAY,CAAC,CAAC;QACnD,MAAM,cAAc,GAAG,IAAA,qBAAY,EACjC,WAAW,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAC1C,CAAC;QAEF,MAAM,IAAI,KAAK,CACb,kEAAkE,eAAe,eAAe,cAAc,KAAK;YACjH,sCAAsC,cAAc,4BAA4B,CACnF,CAAC;IACJ,CAAC;IAGD,MAAM,YAAY,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC,UAAU,CAAC;IAC1D,MAAM,UAAU,GACd,YAAY,CAAC,WAAW,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC;IAE/D,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,CAAC;AACtC,CAAC;AAED,SAAgB,mBAAmB,CACjC,YAAqB,EACrB,mBAA4B;IAE5B,IAAI,mBAAmB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CACb,uEAAuE,CACxE,CAAC;IACJ,CAAC;IAED,IAAI,YAAY,GAAG,CAAC,GAAG,mBAAmB,CAAC,CAAC;IAC5C,IAAI,SAAS,GAAiB,IAAI,CAAC;IAGnC,OAAO,YAAY,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAChC,IAAI,CAAC;YACH,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,GAAG,0BAA0B,CAC7D,YAAY,EACZ,YAAY,CACb,CAAC;YACF,MAAM,gBAAgB,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;YACpD,MAAM,cAAc,GAAG,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC;YACtE,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,gBAAgB,EAAE,cAAc,EAAE,CAAC;QACxE,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,SAAS,GAAG,CAAU,CAAC;YAEvB,YAAY,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IAED,MAAM,IAAI,KAAK,CACb,qIAAqI,SAAS,EAAE,OAAO,EAAE,CAC1J,CAAC;AACJ,CAAC"}
+{"version":3,"file":"patcher.js","sourceRoot":"","sources":["../../../packages/tokenpatch/patcher.ts"],"names":[],"mappings":";;AAiFA,wDAgCC;AA8CD,oDAkCC;AA6HD,kDAsEC;AAnYD,qCAA0D;AAC1D,4FAG8D;AAe9D,MAAM,eAAe,GAAG,CACtB,YAA0B,EAC1B,QAAe,EACR,EAAE;IACT,IAAI,CAAC,YAAY;QAAE,OAAO,QAAQ,CAAC;IAGnC,MAAM,kBAAkB,GAAG,YAAY,CAAC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IACtE,MAAM,cAAc,GAAG,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAE9D,IAAI,kBAAkB,IAAI,CAAC,cAAc,EAAE,CAAC;QAC1C,OAAO,YAAY,CAAC;IACtB,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC;AAGF,SAAS,6BAA6B,CACpC,YAAqB,EACrB,WAAoB;IAEpB,IAAI,YAAY,GAAkB,IAAI,CAAC;IACvC,IAAI,UAAU,GAAkB,IAAI,CAAC;IACrC,IAAI,SAAS,GAAG,EAAE,CAAC;IAEnB,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,IAAI,WAAW,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,CAAC;QACxE,MAAM,YAAY,GAAG,WAAW,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,GAAG,UAAU,CAAC,CAAC;QACxE,MAAM,OAAO,GAAG,IAAA,yBAAgB,EAAC,YAAY,EAAE,YAAY,CAAC,CAAC;QAE7D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,YAAY,GAAG,CAAC,CAAC;YACjB,MAAM,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YAC/B,UAAU,GAAG,YAAY,CAAC,WAAW,GAAG,UAAU,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC;YACjE,MAAM;QACR,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,eAAe,GAAG,IAAA,qBAAY,EAAC,YAAY,CAAC,CAAC;YACnD,MAAM,SAAS,GAAG,OAAO;iBACtB,GAAG,CACF,CAAC,CAAC,EAAE,EAAE,CACJ,QAAQ,YAAY,CAAC,CAAC,CAAC,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAC7F;iBACA,IAAI,CAAC,IAAI,CAAC,CAAC;YACd,SAAS,GAAG,0CAA0C,eAAe,kBAAkB,OAAO,CAAC,MAAM,eAAe,SAAS,GAAG,CAAC;QACnI,CAAC;IACH,CAAC;IAED,IAAI,YAAY,KAAK,IAAI,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;QACjD,MAAM,cAAc,GAAG,WAAW,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACjE,MAAM,IAAI,KAAK,CACb,SAAS;YACP,0CAA0C,IAAA,qBAAY,EACpD,cAAc,CACf,yBAAyB,CAC7B,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,CAAC;AACtC,CAAC;AAED,SAAgB,sBAAsB,CACpC,YAAqB,EACrB,mBAA4B;IAE5B,IAAI,mBAAmB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CACb,iCAAiC,mDAA+B,UAAU,CAC3E,CAAC;IACJ,CAAC;IAED,IAAI,YAAY,GAAG,CAAC,GAAG,mBAAmB,CAAC,CAAC;IAC5C,IAAI,SAAS,GAAiB,IAAI,CAAC;IAGnC,OAAO,YAAY,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAChC,IAAI,CAAC;YACH,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,GAAG,6BAA6B,CAChE,YAAY,EACZ,YAAY,CACb,CAAC;YACF,MAAM,gBAAgB,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;YACpD,MAAM,cAAc,GAAG,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC;YACtE,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,gBAAgB,EAAE,cAAc,EAAE,CAAC;QACxE,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,SAAS,GAAG,eAAe,CAAC,SAAS,EAAE,CAAU,CAAC,CAAC;YACnD,YAAY,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IAED,MAAM,IAAI,KAAK,CACb,mBAAmB,mDAA+B,yFAAyF,SAAS,EAAE,OAAO,EAAE,CAChK,CAAC;AACJ,CAAC;AAGD,SAAS,2BAA2B,CAClC,YAAqB,EACrB,WAAoB,EACpB,UAAkB;IAElB,IAAI,YAAY,GAAkB,IAAI,CAAC;IACvC,IAAI,UAAU,GAAkB,IAAI,CAAC;IACrC,IAAI,SAAS,GAAG,EAAE,CAAC;IAEnB,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,IAAI,WAAW,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,CAAC;QACxE,MAAM,YAAY,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;QACtD,MAAM,OAAO,GAAG,IAAA,yBAAgB,EAAC,YAAY,EAAE,YAAY,CAAC,CAAC;QAE7D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,YAAY,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;YACnD,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC;YAC/B,MAAM;QACR,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,eAAe,GAAG,IAAA,qBAAY,EAAC,YAAY,CAAC,CAAC;YACnD,MAAM,SAAS,GAAG,OAAO;iBACtB,GAAG,CACF,CAAC,CAAC,EAAE,EAAE,CACJ,QAAQ,YAAY,CAAC,CAAC,CAAC,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAC7F;iBACA,IAAI,CAAC,IAAI,CAAC,CAAC;YACd,SAAS,GAAG,0CAA0C,eAAe,kBAAkB,OAAO,CAAC,MAAM,eAAe,SAAS,GAAG,CAAC;QACnI,CAAC;IACH,CAAC;IAED,IAAI,YAAY,KAAK,IAAI,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;QACjD,MAAM,cAAc,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC/C,MAAM,IAAI,KAAK,CACb,SAAS;YACP,0CAA0C,IAAA,qBAAY,EACpD,cAAc,CACf,yBAAyB,CAC7B,CAAC;IACJ,CAAC;IACD,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,CAAC;AACtC,CAAC;AAED,SAAgB,oBAAoB,CAClC,YAAqB,EACrB,mBAA4B,EAC5B,UAAkB;IAElB,IAAI,mBAAmB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CACb,iCAAiC,iDAA6B,UAAU,CACzE,CAAC;IACJ,CAAC;IAED,IAAI,YAAY,GAAG,CAAC,GAAG,mBAAmB,CAAC,CAAC;IAC5C,IAAI,SAAS,GAAiB,IAAI,CAAC;IAGnC,OAAO,YAAY,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAChC,IAAI,CAAC;YACH,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,GAAG,2BAA2B,CAC9D,YAAY,EACZ,YAAY,EACZ,UAAU,CACX,CAAC;YACF,MAAM,gBAAgB,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;YACpD,MAAM,cAAc,GAAG,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC;YACtE,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,gBAAgB,EAAE,cAAc,EAAE,CAAC;QACxE,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,SAAS,GAAG,eAAe,CAAC,SAAS,EAAE,CAAU,CAAC,CAAC;YACnD,YAAY,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IAED,MAAM,IAAI,KAAK,CACb,mBAAmB,iDAA6B,yFAAyF,SAAS,EAAE,OAAO,EAAE,CAC9J,CAAC;AACJ,CAAC;AAQD,SAAS,mBAAmB,CAC1B,YAAqB,EACrB,WAAoB;IAEpB,IAAI,YAAY,GAAmB,IAAI,CAAC;IACxC,IAAI,WAAW,GAAkB,IAAI,CAAC;IACtC,IAAI,eAAe,GAAkB,IAAI,CAAC;IAE1C,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,GAAG,WAAW,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,CAAC;QACvE,MAAM,aAAa,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;QACvD,MAAM,aAAa,GAAG,IAAA,yBAAgB,EAAC,YAAY,EAAE,aAAa,CAAC,CAAC;QAEpE,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,YAAY,GAAG,aAAa,CAAC;YAC7B,WAAW,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;YAC/B,eAAe,GAAG,IAAI,CAAC;YACvB,MAAM;QACR,CAAC;QACD,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAG,IAAA,qBAAY,EAAC,aAAa,CAAC,CAAC;YAC9C,MAAM,SAAS,GAAG,aAAa;iBAC5B,GAAG,CACF,CAAC,CAAC,EAAE,EAAE,CACJ,QAAQ,YAAY,CAAC,CAAC,CAAC,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAC7F;iBACA,IAAI,CAAC,IAAI,CAAC,CAAC;YACd,eAAe,GAAG,0CAA0C,SAAS,kBAAkB,aAAa,CAAC,MAAM,eAAe,SAAS,GAAG,CAAC;QACzI,CAAC;QACD,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,eAAe,EAAE,CAAC;gBACrB,MAAM,SAAS,GAAG,IAAA,qBAAY,EAAC,aAAa,CAAC,CAAC;gBAC9C,eAAe,GAAG,6BAA6B,SAAS,GAAG,CAAC;YAC9D,CAAC;YACD,MAAM;QACR,CAAC;IACH,CAAC;IAED,IAAI,CAAC,YAAY,IAAI,WAAW,KAAK,IAAI,EAAE,CAAC;QAC1C,MAAM,IAAI,KAAK,CAAC,eAAe,IAAI,wCAAwC,CAAC,CAAC;IAC/E,CAAC;IAED,MAAM,YAAY,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC,UAAU,CAAC;IAE1D,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,CAAC;AACrD,CAAC;AAMD,SAAS,mBAAmB,CAC1B,YAAqB,EACrB,WAAoB,EACpB,YAAqB,EACrB,WAAmB;IAEnB,IAAI,YAAY,GAAmB,IAAI,CAAC;IACxC,IAAI,WAAW,GAAkB,IAAI,CAAC;IACtC,IAAI,eAAe,GAAkB,IAAI,CAAC;IAC1C,MAAM,gBAAgB,GAAG,WAAW,GAAG,YAAY,CAAC,MAAM,CAAC;IAC3D,MAAM,iBAAiB,GAAG,YAAY,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;IAE/D,KACE,IAAI,UAAU,GAAG,CAAC,EAClB,UAAU,IAAI,WAAW,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM,EACtD,UAAU,EAAE,EACZ,CAAC;QACD,MAAM,aAAa,GAAG,WAAW,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,GAAG,UAAU,CAAC,CAAC;QACzE,MAAM,aAAa,GAAG,IAAA,yBAAgB,EAAC,iBAAiB,EAAE,aAAa,CAAC,CAAC;QAEzE,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,YAAY,GAAG,aAAa,CAAC;YAC7B,WAAW,GAAG,gBAAgB,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;YAClD,eAAe,GAAG,IAAI,CAAC;YACvB,MAAM;QACR,CAAC;QACD,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAG,IAAA,qBAAY,EAAC,aAAa,CAAC,CAAC;YAC9C,MAAM,SAAS,GAAG,aAAa;iBAC5B,GAAG,CACF,CAAC,CAAC,EAAE,EAAE,CACJ,QACE,YAAY,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC,aAAa,EAAE,GAAG;gBACnD,CAAC,CAAC,YAAY,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC,GAAG,GAAG,CAAC;gBAC1D,CAAC,CAAC,GACN,EAAE,CACL;iBACA,IAAI,CAAC,IAAI,CAAC,CAAC;YACd,eAAe,GAAG,0CAA0C,SAAS,eAAe,aAAa,CAAC,MAAM,4BAA4B,SAAS,GAAG,CAAC;QACnJ,CAAC;IACH,CAAC;IACD,IAAI,CAAC,YAAY,IAAI,WAAW,KAAK,IAAI,EAAE,CAAC;QAC1C,IAAI,eAAe,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC;QACnC,CAAC;QACD,MAAM,cAAc,GAAG,QACrB,YAAY,CAAC,WAAW,CAAC,CAAC,aAAa,EAAE,GAAG;YAC1C,CAAC,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC,aAAa,CAAC,GAAG,GAAG,CAAC;YACjD,CAAC,CAAC,GACN,EAAE,CAAC;QACH,MAAM,eAAe,GAAG,IAAA,qBAAY,EAAC,YAAY,CAAC,CAAC;QACnD,MAAM,cAAc,GAAG,IAAA,qBAAY,EACjC,WAAW,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAC1C,CAAC;QAEF,MAAM,IAAI,KAAK,CACb,kEAAkE,eAAe,eAAe,cAAc,KAAK;YACjH,sCAAsC,cAAc,4BAA4B,CACnF,CAAC;IACJ,CAAC;IAED,MAAM,UAAU,GACd,YAAY,CAAC,WAAW,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC;IAE/D,OAAO,EAAE,UAAU,EAAE,CAAC;AACxB,CAAC;AAED,SAAgB,mBAAmB,CACjC,YAAqB,EACrB,mBAA4B;IAE5B,IAAI,mBAAmB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CACb,uEAAuE,CACxE,CAAC;IACJ,CAAC;IAED,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,OAAO,GAAG,CAAC,CAAC;IAClB,IAAI,SAAS,GAAiB,IAAI,CAAC;IAEnC,IAAI,UAAU,GAAwB,IAAI,CAAC;IAG3C,IAAI,YAAY,GAAG,CAAC,GAAG,mBAAmB,CAAC,CAAC;IAC5C,OAAO,YAAY,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAChC,IAAI,CAAC;YACH,UAAU,GAAG,mBAAmB,CAAC,YAAY,EAAE,YAAY,CAAC,CAAC;YAE7D,SAAS,GAAG,mBAAmB,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;YAC7D,MAAM;QACR,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,SAAS,GAAG,eAAe,CAAC,SAAS,EAAE,CAAU,CAAC,CAAC;YAEnD,YAAY,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IAED,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CACb,+GAA+G,SAAS,EAAE,OAAO,EAAE,CACpI,CAAC;IACJ,CAAC;IAID,YAAY,GAAG,mBAAmB,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IACpD,SAAS,GAAG,IAAI,CAAC;IAEjB,OAAO,YAAY,CAAC,MAAM,IAAI,UAAU,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAEjE,IAAI,CAAC;YACH,MAAM,EAAE,UAAU,EAAE,GAAG,mBAAmB,CACxC,YAAY,EACZ,YAAY,EACZ,UAAU,CAAC,YAAY,EACvB,UAAU,CAAC,WAAW,CACvB,CAAC;YAEF,MAAM,gBAAgB,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;YACpD,MAAM,cAAc,GAAG,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC;YACtE,OAAO;gBACL,YAAY,EAAE,UAAU,CAAC,YAAY;gBACrC,UAAU;gBACV,gBAAgB;gBAChB,cAAc;aACf,CAAC;QACJ,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,SAAS,GAAG,eAAe,CAAC,SAAS,EAAE,CAAU,CAAC,CAAC;YAEnD,YAAY,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IAED,MAAM,IAAI,KAAK,CACb,+GAA+G,SAAS,EAAE,OAAO,EAAE,CACpI,CAAC;AACJ,CAAC"}

package/backend/dist/packages/tokenpatch/strategies/tiktoken-tokenizer.d.ts

@@ -0,0 +1,6 @@
+import { Token } from '../types';
+import { TokenizerStrategy } from '../tokenizer.interface';
+export declare class TiktokenTokenizer implements TokenizerStrategy {
+    private enc;
+    tokenize(content: string): Token[];
+}

package/backend/dist/packages/tokenpatch/strategies/tiktoken-tokenizer.js

@@ -0,0 +1,28 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TiktokenTokenizer = void 0;
+const js_tiktoken_1 = require("js-tiktoken");
+class TiktokenTokenizer {
+    constructor() {
+        this.enc = (0, js_tiktoken_1.getEncoding)('cl100k_base');
+    }
+    tokenize(content) {
+        const tokens = [];
+        const encoded = this.enc.encode(content);
+        let currentIndex = 0;
+        for (const tokenId of encoded) {
+            const text = this.enc.decode([tokenId]);
+            const length = text.length;
+            tokens.push({
+                text,
+                type: 'bpe',
+                startIndex: currentIndex,
+                endIndex: currentIndex + length,
+            });
+            currentIndex += length;
+        }
+        return tokens;
+    }
+}
+exports.TiktokenTokenizer = TiktokenTokenizer;
+//# sourceMappingURL=tiktoken-tokenizer.js.map

package/backend/dist/packages/tokenpatch/strategies/tiktoken-tokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tiktoken-tokenizer.js","sourceRoot":"","sources":["../../../../packages/tokenpatch/strategies/tiktoken-tokenizer.ts"],"names":[],"mappings":";;;AAAA,6CAA0C;AAI1C,MAAa,iBAAiB;IAA9B;QAEU,QAAG,GAAG,IAAA,yBAAW,EAAC,aAAa,CAAC,CAAC;IA4B3C,CAAC;IA1BC,QAAQ,CAAC,OAAe;QACtB,MAAM,MAAM,GAAY,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAEzC,IAAI,YAAY,GAAG,CAAC,CAAC;QAIrB,KAAK,MAAM,OAAO,IAAI,OAAO,EAAE,CAAC;YAG5B,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YACxC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;YAE3B,MAAM,CAAC,IAAI,CAAC;gBACR,IAAI;gBACJ,IAAI,EAAE,KAAK;gBACX,UAAU,EAAE,YAAY;gBACxB,QAAQ,EAAE,YAAY,GAAG,MAAM;aAGlC,CAAC,CAAC;YACH,YAAY,IAAI,MAAM,CAAC;QAC3B,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AA9BD,8CA8BC"}

package/backend/dist/packages/tokenpatch/strategies/tree-sitter-tokenizer.d.ts

@@ -0,0 +1,9 @@
+import { Token } from '../types';
+import { TokenizerStrategy } from '../tokenizer.interface';
+import type { Parser } from 'web-tree-sitter';
+export declare class TreeSitterTokenizer implements TokenizerStrategy {
+    private parser;
+    constructor(parser: Parser);
+    tokenize(content: string): Token[];
+    private collectTokens;
+}

package/backend/dist/packages/tokenpatch/strategies/tree-sitter-tokenizer.js

@@ -0,0 +1,36 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TreeSitterTokenizer = void 0;
+class TreeSitterTokenizer {
+    constructor(parser) {
+        this.parser = parser;
+    }
+    tokenize(content) {
+        const tree = this.parser.parse(content);
+        return this.collectTokens(tree, content);
+    }
+    collectTokens(tree, code) {
+        const tokens = [];
+        function visit(node) {
+            if (node.childCount === 0) {
+                tokens.push({
+                    text: code.slice(node.startIndex, node.endIndex),
+                    type: node.type,
+                    startIndex: node.startIndex,
+                    endIndex: node.endIndex,
+                    startPosition: node.startPosition,
+                });
+                return;
+            }
+            for (let i = 0; i < node.childCount; i++) {
+                const child = node.child(i);
+                if (child)
+                    visit(child);
+            }
+        }
+        visit(tree.rootNode);
+        return tokens;
+    }
+}
+exports.TreeSitterTokenizer = TreeSitterTokenizer;
+//# sourceMappingURL=tree-sitter-tokenizer.js.map

package/backend/dist/packages/tokenpatch/strategies/tree-sitter-tokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tree-sitter-tokenizer.js","sourceRoot":"","sources":["../../../../packages/tokenpatch/strategies/tree-sitter-tokenizer.ts"],"names":[],"mappings":";;;AAIA,MAAa,mBAAmB;IAC9B,YAAoB,MAAc;QAAd,WAAM,GAAN,MAAM,CAAQ;IAAG,CAAC;IAEtC,QAAQ,CAAC,OAAe;QACtB,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACxC,OAAO,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC3C,CAAC;IAEO,aAAa,CAAC,IAAU,EAAE,IAAY;QAC5C,MAAM,MAAM,GAAY,EAAE,CAAC;QAE3B,SAAS,KAAK,CAAC,IAAU;YACvB,IAAI,IAAI,CAAC,UAAU,KAAK,CAAC,EAAE,CAAC;gBAC1B,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC;oBAChD,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,UAAU,EAAE,IAAI,CAAC,UAAU;oBAC3B,QAAQ,EAAE,IAAI,CAAC,QAAQ;oBACvB,aAAa,EAAE,IAAI,CAAC,aAAa;iBAClC,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;gBACzC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC5B,IAAI,KAAK;oBAAE,KAAK,CAAC,KAAK,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACrB,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAhCD,kDAgCC"}

package/backend/dist/packages/tokenpatch/tokenizer.interface.d.ts

@@ -0,0 +1,4 @@
+import { Token } from './types';
+export interface TokenizerStrategy {
+    tokenize(content: string): Token[];
+}

package/backend/dist/packages/tokenpatch/tokenizer.interface.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=tokenizer.interface.js.map

package/backend/dist/packages/tokenpatch/tokenizer.interface.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.interface.js","sourceRoot":"","sources":["../../../packages/tokenpatch/tokenizer.interface.ts"],"names":[],"mappings":""}

package/backend/dist/packages/tokenpatch/tokens.d.ts

@@ -1,6 +1,4 @@
-import type { Tree } from 'web-tree-sitter';
 import type { Token } from './types';
-export declare function collectTokens(tree: Tree, code: string): Token[];
 export declare function tokensEqual(a: Token, b: Token): boolean;
 export declare function formatAnchor(anchor: Token[]): string;
 export declare function findAllSequences(haystack: Token[], needle: Token[]): number[];

package/backend/dist/packages/tokenpatch/tokens.js

@@ -1,32 +1,13 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.collectTokens = collectTokens;
 exports.tokensEqual = tokensEqual;
 exports.formatAnchor = formatAnchor;
 exports.findAllSequences = findAllSequences;
-function collectTokens(tree, code) {
-    const tokens = [];
-    function visit(node) {
-        if (node.childCount === 0) {
-            tokens.push({
-                text: code.slice(node.startIndex, node.endIndex),
-                type: node.type,
-                startIndex: node.startIndex,
-                endIndex: node.endIndex,
-                startPosition: node.startPosition,
-            });
-            return;
-        }
-        for (let i = 0; i < node.childCount; i++) {
-            const child = node.child(i);
-            if (child)
-                visit(child);
-        }
-    }
-    visit(tree.rootNode);
-    return tokens;
-}
 function tokensEqual(a, b) {
+    if (a.type === 'bpe' || b.type === 'bpe') {
+        return (a.text.replace(/\r?\n/g, '').trim() ===
+            b.text.replace(/\r?\n/g, '').trim());
+    }
     return a.text === b.text;
 }
 function formatAnchor(anchor) {

package/backend/dist/packages/tokenpatch/tokens.js.map

@@ -1 +1 @@
-{"version":3,"file":"tokens.js","sourceRoot":"","sources":["../../../packages/tokenpatch/tokens.ts"],"names":[],"mappings":";;AAGA,sCAwBC;AAED,kCAEC;AAED,oCAEC;AAED,4CAYC;AA9CD,SAAgB,aAAa,CAAC,IAAU,EAAE,IAAY;IACpD,MAAM,MAAM,GAAY,EAAE,CAAC;IAE3B,SAAS,KAAK,CAAC,IAAU;QACvB,IAAI,IAAI,CAAC,UAAU,KAAK,CAAC,EAAE,CAAC;YAC1B,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC;gBAChD,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,aAAa,EAAE,IAAI,CAAC,aAAa;aAClC,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAGD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YACzC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC5B,IAAI,KAAK;gBAAE,KAAK,CAAC,KAAK,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACrB,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAgB,WAAW,CAAC,CAAQ,EAAE,CAAQ;IAC5C,OAAO,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,CAAC;AAC3B,CAAC;AAED,SAAgB,YAAY,CAAC,MAAe;IAC1C,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC7C,CAAC;AAED,SAAgB,gBAAgB,CAAC,QAAiB,EAAE,MAAe;IACjE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACnC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,EAAE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACjE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC7C,SAAS,KAAK,CAAC;YACjB,CAAC;QACH,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}
+{"version":3,"file":"tokens.js","sourceRoot":"","sources":["../../../packages/tokenpatch/tokens.ts"],"names":[],"mappings":";;AAEA,kCAWC;AAED,oCAEC;AAED,4CAYC;AA7BD,SAAgB,WAAW,CAAC,CAAQ,EAAE,CAAQ;IAI5C,IAAI,CAAC,CAAC,IAAI,KAAK,KAAK,IAAI,CAAC,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;QACzC,OAAO,CACL,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE;YACnC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CACpC,CAAC;IACJ,CAAC;IACD,OAAO,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,CAAC;AAC3B,CAAC;AAED,SAAgB,YAAY,CAAC,MAAe;IAC1C,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC7C,CAAC;AAED,SAAgB,gBAAgB,CAAC,QAAiB,EAAE,MAAe;IACjE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACnC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,EAAE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACjE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC7C,SAAS,KAAK,CAAC;YACjB,CAAC;QACH,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/backend/dist/packages/tokenpatch/types.d.ts

@@ -1,8 +1,8 @@
 import type { Point } from 'web-tree-sitter';
 export interface Token {
     text: string;
-    type: string;
+    type?: string;
     startIndex: number;
     endIndex: number;
-    startPosition: Point;
+    startPosition?: Point;
 }