repoburg 1.3.9 → 1.3.11

package/backend/packages/tokenpatch/patcher.spec.ts

@@ -0,0 +1,194 @@
+import {
+  handleBeginOfFilePatch,
+  handleEndOfFilePatch,
+  handleStandardPatch,
+} from './patcher';
+import type { Token } from './types';
+
+// Helper to create mock tokens for testing.
+// Simplified: we only care about text, startIndex, and endIndex for these tests.
+const makeTokens = (texts: string[]): Token[] => {
+  let offset = 0;
+  return texts.map((text) => {
+    const start = offset;
+    const end = start + text.length;
+    offset = end + 1; // Simulate space
+    return {
+      text,
+      type: 'test',
+      startIndex: start,
+      endIndex: end,
+      startPosition: { row: 0, column: start },
+    };
+  });
+};
+
+describe('patcher', () => {
+  describe('handleStandardPatch', () => {
+    it('should find a unique prefix and suffix', () => {
+      const sourceTokens = makeTokens(['prefix', '(', 'old', ')', 'suffix']);
+      const patchTokens = makeTokens(['prefix', '(', 'new', ')', 'suffix']);
+      const result = handleStandardPatch(sourceTokens, patchTokens);
+      expect(result).toEqual({
+        patchInsertEnd: 21,
+        patchInsertStart: 0,
+        replaceStart: 0, // 'prefix' startIndex
+        replaceEnd: 21, // 'suffix' endIndex
+      });
+    });
+
+    it('should trim non-matching tokens from both ends to find a match', () => {
+      const sourceTokens = makeTokens(['prefix', 'content', 'suffix']);
+      // The patch has extra tokens at the start and end that aren't in the source.
+      const patchTokens = makeTokens([
+        'bad', // 0-3
+        'prefix', // 4-10
+        'new-content', // 11-22
+        'suffix', // 23-29
+        'good', // 30-34
+      ]);
+      const result = handleStandardPatch(sourceTokens, patchTokens);
+      // It should fail with 'bad'/'good', then trim them, then succeed on ['prefix', 'new-content', 'suffix']
+      expect(result).toEqual({
+        replaceStart: 0, // start of 'prefix' in source
+        replaceEnd: 21, // end of 'suffix' in source
+        patchInsertStart: 4, // start of 'prefix' in patch
+        patchInsertEnd: 29, // end of 'suffix' in patch
+      });
+    });
+
+    it('should throw for persistently ambiguous prefix', () => {
+      const sourceTokens = makeTokens(['a', 'b', 'c', 'a', 'b', 'd']);
+      const patchTokens = makeTokens(['a', 'b', 'e']); // a b is ambiguous
+      expect(() => handleStandardPatch(sourceTokens, patchTokens)).toThrow(
+        /Ambiguous prefix anchor/,
+      );
+    });
+
+    it('should throw for persistently ambiguous suffix', () => {
+      const sourceTokens = makeTokens(['prefix', 'x', 'suffix', 'y', 'suffix']);
+      const patchTokens = makeTokens(['prefix', 'z', 'suffix']);
+      expect(() => handleStandardPatch(sourceTokens, patchTokens)).toThrow(
+        /Ambiguous suffix anchor/,
+      );
+    });
+
+    it('should throw if patch has fewer than two tokens', () => {
+      const sourceTokens = makeTokens(['a', 'b', 'c']);
+      const patchTokens = makeTokens(['a']);
+      expect(() => handleStandardPatch(sourceTokens, patchTokens)).toThrow(
+        'Patch must contain at least two tokens to form a prefix and a suffix.',
+      );
+    });
+  });
+
+  describe('handleBeginOfFilePatch', () => {
+    it('should find a unique suffix anchor', () => {
+      const sourceTokens = makeTokens(['a', 'b', 'c', 'd']);
+      const patchTokens = makeTokens(['x', 'y', 'c', 'd']); // Suffix is 'c', 'd'
+      const result = handleBeginOfFilePatch(sourceTokens, patchTokens);
+      expect(result).toEqual({
+        patchInsertEnd: 7,
+        patchInsertStart: 0,
+        replaceStart: 0,
+        replaceEnd: 7, // endIndex of 'd'
+      });
+    });
+
+    it('should trim from the start to find a unique suffix', () => {
+      // The "good" part of the patch, ['a', 'b'], is unique in the source.
+      const sourceTokens = makeTokens(['x', 'y', 'a', 'b']);
+      // The initial patch will fail because the suffix ['y', 'a', 'b'] is not in the source.
+      const patchTokens = makeTokens([
+        'bad', // 0-3
+        'token', // 4-9
+        'a', // 10-11
+        'b', // 12-13
+      ]);
+      const result = handleBeginOfFilePatch(sourceTokens, patchTokens);
+      // Should fail, trim 'bad', fail, trim 'token', then succeed with suffix ['a', 'b'].
+      expect(result).toEqual({
+        replaceStart: 0,
+        replaceEnd: 7, // end of 'b' in source
+        patchInsertStart: 0, // start of 'a' in patch
+        patchInsertEnd: 13, // end of 'b' in patch
+      });
+    });
+
+    it('should throw for a persistently ambiguous suffix', () => {
+      const sourceTokens = makeTokens(['a', 'b', 'a', 'b']);
+      const patchTokens = makeTokens(['x', 'y', 'a', 'b']);
+      expect(() => handleBeginOfFilePatch(sourceTokens, patchTokens)).toThrow(
+        /Ambiguous suffix anchor/,
+      );
+    });
+
+    it('should throw if anchor not found even after trimming', () => {
+      const sourceTokens = makeTokens(['a', 'b', 'c']);
+      const patchTokens = makeTokens(['x', 'y', 'z']);
+      expect(() => handleBeginOfFilePatch(sourceTokens, patchTokens)).toThrow(
+        /Suffix anchor not found/,
+      );
+    });
+  });
+
+  describe('handleEndOfFilePatch', () => {
+    const sourceCode = 'a b c d'; // length 7
+    const sourceTokens = makeTokens(['a', 'b', 'c', 'd']);
+
+    it('should find a unique prefix anchor', () => {
+      const patchTokens = makeTokens(['a', 'b', 'x', 'y']); // Prefix is 'a', 'b'
+      const result = handleEndOfFilePatch(
+        sourceTokens,
+        patchTokens,
+        sourceCode,
+      );
+      expect(result).toEqual({
+        patchInsertEnd: 7,
+        patchInsertStart: 0,
+        replaceStart: 0, // startIndex of 'a'
+        replaceEnd: 7, // sourceCode.length
+      });
+    });
+
+    it('should trim from the end to find a unique prefix', () => {
+      // The "good" part of the patch, ['c', 'd'], is unique in the source.
+      const sourceTokens = makeTokens(['c', 'd', 'x', 'y']);
+      const sourceCode = 'c d x y'; // length 7
+      // The initial patch will fail because the prefix ['c', 'd', 'bad'] is not in the source.
+      const patchTokens = makeTokens([
+        'c', // 0-1
+        'd', // 2-3
+        'bad', // 4-7
+        'token', // 8-13
+      ]);
+      const result = handleEndOfFilePatch(
+        sourceTokens,
+        patchTokens,
+        sourceCode,
+      );
+      // Should fail, trim 'token', fail, trim 'bad', then succeed with prefix ['c', 'd'].
+      expect(result).toEqual({
+        replaceStart: 0, // start of 'c' in source
+        replaceEnd: 7, // sourceCode.length
+        patchInsertStart: 0, // start of 'c' in patch
+        patchInsertEnd: 13, // end of 'd' in patch
+      });
+    });
+
+    it('should throw for a persistently ambiguous prefix', () => {
+      const ambiguousSourceTokens = makeTokens(['a', 'b', 'a', 'b']);
+      const patchTokens = makeTokens(['a', 'b', 'x', 'y']);
+      expect(() =>
+        handleEndOfFilePatch(ambiguousSourceTokens, patchTokens, 'a b a b'),
+      ).toThrow(/Ambiguous prefix anchor/);
+    });
+
+    it('should throw if anchor not found even after trimming', () => {
+      const patchTokens = makeTokens(['x', 'y', 'z']);
+      expect(() =>
+        handleEndOfFilePatch(sourceTokens, patchTokens, sourceCode),
+      ).toThrow(/Prefix anchor not found/);
+    });
+  });
+});

package/backend/packages/tokenpatch/patcher.ts

@@ -0,0 +1,300 @@
+import type { Token } from './types';
+import { findAllSequences, formatAnchor } from './tokens';
+
+interface PatchResult {
+  replaceStart: number;
+  replaceEnd: number;
+  patchInsertStart: number;
+  patchInsertEnd: number;
+}
+
+interface SimplePatchResult {
+  replaceStart: number;
+  replaceEnd: number;
+}
+
+// Internal helper for the original matching logic
+function _findBeginOfFilePatchLocation(
+  sourceTokens: Token[],
+  patchTokens: Token[],
+): SimplePatchResult {
+  let replaceStart: number | null = null;
+  let replaceEnd: number | null = null;
+  let lastError = '';
+
+  for (let anchorSize = 1; anchorSize <= patchTokens.length; anchorSize++) {
+    const suffixAnchor = patchTokens.slice(patchTokens.length - anchorSize);
+    const indices = findAllSequences(sourceTokens, suffixAnchor);
+
+    if (indices.length === 1) {
+      replaceStart = 0;
+      const suffixIndex = indices[0];
+      replaceEnd = sourceTokens[suffixIndex + anchorSize - 1].endIndex;
+      break; // Found unique anchor
+    }
+
+    if (indices.length > 1) {
+      const formattedAnchor = formatAnchor(suffixAnchor);
+      const locations = indices
+        .map((i) => `line ${sourceTokens[i].startPosition.row + 1}`)
+        .join(', ');
+      lastError = `Ambiguous suffix anchor. The sequence "${formattedAnchor}" was found at ${indices.length} locations: ${locations}.`;
+    }
+  }
+
+  if (replaceStart === null || replaceEnd === null) {
+    const smallestAnchor = patchTokens.slice(patchTokens.length - 1);
+    throw new Error(
+      lastError ||
+        `Suffix anchor not found. The sequence "${formatAnchor(
+          smallestAnchor,
+        )}" could not be located.`,
+    );
+  }
+
+  return { replaceStart, replaceEnd };
+}
+
+export function handleBeginOfFilePatch(
+  sourceTokens: Token[],
+  originalPatchTokens: Token[],
+): PatchResult {
+  if (originalPatchTokens.length === 0) {
+    throw new Error('Patch is empty after removing @begin-of-file marker.');
+  }
+
+  let patchAttempt = [...originalPatchTokens];
+  let lastError: Error | null = null;
+
+  // Outer loop for trimming tokens from the start
+  while (patchAttempt.length >= 1) {
+    try {
+      const { replaceStart, replaceEnd } = _findBeginOfFilePatchLocation(
+        sourceTokens,
+        patchAttempt,
+      );
+      const patchInsertStart = patchAttempt[0].startIndex;
+      const patchInsertEnd = patchAttempt[patchAttempt.length - 1].endIndex;
+      return { replaceStart, replaceEnd, patchInsertStart, patchInsertEnd };
+    } catch (e) {
+      lastError = e as Error;
+      patchAttempt = patchAttempt.slice(1); // Trim one token from the beginning
+    }
+  }
+
+  throw new Error(
+    `Failed to apply @begin-of-file patch. Could not find a unique anchor, even after trimming tokens. Last known error: ${lastError?.message}`,
+  );
+}
+
+// Internal helper for the original matching logic
+function _findEndOfFilePatchLocation(
+  sourceTokens: Token[],
+  patchTokens: Token[],
+  sourceCode: string,
+): SimplePatchResult {
+  let replaceStart: number | null = null;
+  let replaceEnd: number | null = null;
+  let lastError = '';
+
+  for (let anchorSize = 1; anchorSize <= patchTokens.length; anchorSize++) {
+    const prefixAnchor = patchTokens.slice(0, anchorSize);
+    const indices = findAllSequences(sourceTokens, prefixAnchor);
+
+    if (indices.length === 1) {
+      replaceStart = sourceTokens[indices[0]].startIndex;
+      replaceEnd = sourceCode.length;
+      break; // Found unique anchor
+    }
+
+    if (indices.length > 1) {
+      const formattedAnchor = formatAnchor(prefixAnchor);
+      const locations = indices
+        .map((i) => `line ${sourceTokens[i].startPosition.row + 1}`)
+        .join(', ');
+      lastError = `Ambiguous prefix anchor. The sequence "${formattedAnchor}" was found at ${indices.length} locations: ${locations}.`;
+    }
+  }
+
+  if (replaceStart === null || replaceEnd === null) {
+    const smallestAnchor = patchTokens.slice(0, 1);
+    throw new Error(
+      lastError ||
+        `Prefix anchor not found. The sequence "${formatAnchor(
+          smallestAnchor,
+        )}" could not be located.`,
+    );
+  }
+  return { replaceStart, replaceEnd };
+}
+
+export function handleEndOfFilePatch(
+  sourceTokens: Token[],
+  originalPatchTokens: Token[],
+  sourceCode: string,
+): PatchResult {
+  if (originalPatchTokens.length === 0) {
+    throw new Error('Patch is empty after removing @end-of-file marker.');
+  }
+
+  let patchAttempt = [...originalPatchTokens];
+  let lastError: Error | null = null;
+
+  // Outer loop for trimming tokens from the end
+  while (patchAttempt.length >= 1) {
+    try {
+      const { replaceStart, replaceEnd } = _findEndOfFilePatchLocation(
+        sourceTokens,
+        patchAttempt,
+        sourceCode,
+      );
+      const patchInsertStart = patchAttempt[0].startIndex;
+      const patchInsertEnd = patchAttempt[patchAttempt.length - 1].endIndex;
+      return { replaceStart, replaceEnd, patchInsertStart, patchInsertEnd };
+    } catch (e) {
+      lastError = e as Error;
+      patchAttempt = patchAttempt.slice(0, -1); // Trim one token from the end
+    }
+  }
+
+  throw new Error(
+    `Failed to apply @end-of-file patch. Could not find a unique anchor, even after trimming tokens. Last known error: ${lastError?.message}`,
+  );
+}
+
+// Internal helper for the original matching logic
+function _findStandardPatchLocation(
+  sourceTokens: Token[],
+  patchTokens: Token[],
+): SimplePatchResult {
+  // 1. Find smallest unique prefix
+  let prefixAnchor: Token[] | null = null;
+  let prefixIndex: number | null = null;
+  let bestPrefixError: string | null = null;
+
+  for (let prefixSize = 1; prefixSize < patchTokens.length; prefixSize++) {
+    const currentPrefix = patchTokens.slice(0, prefixSize);
+    const prefixIndices = findAllSequences(sourceTokens, currentPrefix);
+
+    if (prefixIndices.length === 1) {
+      prefixAnchor = currentPrefix;
+      prefixIndex = prefixIndices[0];
+      bestPrefixError = null; // Found it
+      break;
+    }
+    if (prefixIndices.length > 1) {
+      const formatted = formatAnchor(currentPrefix);
+      const locations = prefixIndices
+        .map((i) => `line ${sourceTokens[i].startPosition.row + 1}`)
+        .join(', ');
+      bestPrefixError = `Ambiguous prefix anchor. The sequence "${formatted}" was found at ${prefixIndices.length} locations: ${locations}.`;
+    }
+    if (prefixIndices.length === 0) {
+      if (!bestPrefixError) {
+        const formatted = formatAnchor(currentPrefix);
+        bestPrefixError = `Prefix anchor not found: "${formatted}"`;
+      }
+      break; // A larger prefix will also not be found
+    }
+  }
+
+  if (!prefixAnchor || prefixIndex === null) {
+    throw new Error(
+      bestPrefixError || 'Could not find a unique prefix anchor.',
+    );
+  }
+
+  // 2. Find smallest unique suffix after prefix
+  let suffixAnchor: Token[] | null = null;
+  let suffixIndex: number | null = null;
+  let bestSuffixError: string | null = null;
+  const searchStartIndex = prefixIndex + prefixAnchor.length;
+  const sourceAfterPrefix = sourceTokens.slice(searchStartIndex);
+
+  for (
+    let suffixSize = 1;
+    suffixSize <= patchTokens.length - prefixAnchor.length;
+    suffixSize++
+  ) {
+    const currentSuffix = patchTokens.slice(patchTokens.length - suffixSize);
+    const suffixIndices = findAllSequences(sourceAfterPrefix, currentSuffix);
+
+    if (suffixIndices.length === 1) {
+      suffixAnchor = currentSuffix;
+      suffixIndex = searchStartIndex + suffixIndices[0];
+      bestSuffixError = null; // Found it
+      break;
+    }
+    if (suffixIndices.length > 1) {
+      const formatted = formatAnchor(currentSuffix);
+      const locations = suffixIndices
+        .map(
+          (i) =>
+            `line ${
+              sourceTokens[searchStartIndex + i].startPosition.row + 1
+            }`,
+        )
+        .join(', ');
+      bestSuffixError = `Ambiguous suffix anchor. The sequence "${formatted}" was found ${suffixIndices.length} times after the prefix: ${locations}.`;
+    }
+  }
+  if (!suffixAnchor || suffixIndex === null) {
+    if (bestSuffixError) {
+      throw new Error(bestSuffixError);
+    }
+    const prefixLocation = `line ${
+      sourceTokens[prefixIndex].startPosition.row + 1
+    }`;
+    const formattedPrefix = formatAnchor(prefixAnchor);
+    const smallestSuffix = formatAnchor(
+      patchTokens.slice(patchTokens.length - 1),
+    );
+
+    throw new Error(
+      `Could not find a unique suffix anchor after the prefix anchor "${formattedPrefix}" (found at ${prefixLocation}). ` +
+        `The smallest suffix searched for ("${smallestSuffix}") was not found after it.`,
+    );
+  }
+
+  // 3. Apply patch
+  const replaceStart = sourceTokens[prefixIndex].startIndex;
+  const replaceEnd =
+    sourceTokens[suffixIndex + suffixAnchor.length - 1].endIndex;
+
+  return { replaceStart, replaceEnd };
+}
+
+export function handleStandardPatch(
+  sourceTokens: Token[],
+  originalPatchTokens: Token[],
+): PatchResult {
+  if (originalPatchTokens.length < 2) {
+    throw new Error(
+      'Patch must contain at least two tokens to form a prefix and a suffix.',
+    );
+  }
+
+  let patchAttempt = [...originalPatchTokens];
+  let lastError: Error | null = null;
+
+  // Outer loop for trimming tokens from both ends
+  while (patchAttempt.length >= 2) {
+    try {
+      const { replaceStart, replaceEnd } = _findStandardPatchLocation(
+        sourceTokens,
+        patchAttempt,
+      );
+      const patchInsertStart = patchAttempt[0].startIndex;
+      const patchInsertEnd = patchAttempt[patchAttempt.length - 1].endIndex;
+      return { replaceStart, replaceEnd, patchInsertStart, patchInsertEnd };
+    } catch (e) {
+      lastError = e as Error;
+      // Trim one token from the start and one from the end for the next attempt
+      patchAttempt = patchAttempt.slice(1, -1);
+    }
+  }
+
+  throw new Error(
+    `Failed to apply patch. Could not find a unique anchor in the source file, even after trimming ambiguous tokens. Last known error: ${lastError?.message}`,
+  );
+}

package/backend/packages/tokenpatch/tokens.spec.ts

@@ -0,0 +1,84 @@
+import { findAllSequences, tokensEqual, formatAnchor } from './tokens';
+import type { Token } from './types';
+
+// A helper to create mock tokens for testing
+const makeTokens = (texts: string[]): Token[] => {
+  let startIndex = 0;
+  return texts.map((text) => {
+    const token: Token = {
+      text,
+      type: 'identifier', // type and positions are not important for these tests
+      startIndex: startIndex,
+      endIndex: startIndex + text.length,
+      startPosition: { row: 0, column: startIndex },
+    };
+    startIndex += text.length + 1; // +1 for space
+    return token;
+  });
+};
+
+describe('token-utils', () => {
+  describe('findAllSequences', () => {
+    it('should find a single unique sequence', () => {
+      const haystack = makeTokens(['a', 'b', 'c', 'd', 'e']);
+      const needle = makeTokens(['b', 'c']);
+      const indices = findAllSequences(haystack, needle);
+      expect(indices).toEqual([1]);
+    });
+
+    it('should find multiple occurrences of a sequence', () => {
+      const haystack = makeTokens(['a', 'b', 'c', 'a', 'b', 'c']);
+      const needle = makeTokens(['a', 'b', 'c']);
+      const indices = findAllSequences(haystack, needle);
+      expect(indices).toEqual([0, 3]);
+    });
+
+    it('should return an empty array if sequence is not found', () => {
+      const haystack = makeTokens(['a', 'b', 'c']);
+      const needle = makeTokens(['d', 'e']);
+      const indices = findAllSequences(haystack, needle);
+      expect(indices).toEqual([]);
+    });
+
+    it('should return an empty array for an empty needle', () => {
+      const haystack = makeTokens(['a', 'b', 'c']);
+      const needle = makeTokens([]);
+      const indices = findAllSequences(haystack, needle);
+      expect(indices).toEqual([]);
+    });
+
+    it('should handle sequences at the beginning and end of haystack', () => {
+      const haystack = makeTokens(['a', 'b', 'c', 'd', 'a', 'b']);
+      const needleAtStart = makeTokens(['a', 'b']);
+      const needleAtEnd = makeTokens(['a', 'b']);
+      expect(findAllSequences(haystack, needleAtStart)).toEqual([0, 4]);
+      expect(findAllSequences(haystack, needleAtEnd)).toEqual([0, 4]);
+    });
+  });
+
+  describe('tokensEqual', () => {
+    it('should return true for tokens with same text', () => {
+      const tokenA = makeTokens(['hello'])[0];
+      const tokenB = makeTokens(['hello'])[0];
+      expect(tokensEqual(tokenA, tokenB)).toBe(true);
+    });
+
+    it('should return false for tokens with different text', () => {
+      const tokenA = makeTokens(['hello'])[0];
+      const tokenB = makeTokens(['world'])[0];
+      expect(tokensEqual(tokenA, tokenB)).toBe(false);
+    });
+  });
+
+  describe('formatAnchor', () => {
+    it('should format an anchor to a space-separated string', () => {
+      const anchor = makeTokens(['const', 'x', '=', '1', ';']);
+      expect(formatAnchor(anchor)).toBe('const x = 1 ;');
+    });
+
+    it('should return an empty string for an empty anchor', () => {
+      const anchor = makeTokens([]);
+      expect(formatAnchor(anchor)).toBe('');
+    });
+  });
+});

package/backend/packages/tokenpatch/tokens.ts

@@ -0,0 +1,50 @@
+import type { Tree, Node } from 'web-tree-sitter';
+import type { Token } from './types';
+
+export function collectTokens(tree: Tree, code: string): Token[] {
+  const tokens: Token[] = [];
+
+  function visit(node: Node) {
+    if (node.childCount === 0) {
+      tokens.push({
+        text: code.slice(node.startIndex, node.endIndex),
+        type: node.type,
+        startIndex: node.startIndex,
+        endIndex: node.endIndex,
+        startPosition: node.startPosition,
+      });
+      return;
+    }
+    // By iterating over all children (not just named), we include punctuation
+    // which is critical for anchor-based matching.
+    for (let i = 0; i < node.childCount; i++) {
+      const child = node.child(i);
+      if (child) visit(child);
+    }
+  }
+
+  visit(tree.rootNode);
+  return tokens;
+}
+
+export function tokensEqual(a: Token, b: Token): boolean {
+  return a.text === b.text;
+}
+
+export function formatAnchor(anchor: Token[]): string {
+  return anchor.map((t) => t.text).join(' ');
+}
+
+export function findAllSequences(haystack: Token[], needle: Token[]): number[] {
+  if (needle.length === 0) return [];
+  const indices: number[] = [];
+  outer: for (let i = 0; i <= haystack.length - needle.length; i++) {
+    for (let j = 0; j < needle.length; j++) {
+      if (!tokensEqual(haystack[i + j], needle[j])) {
+        continue outer;
+      }
+    }
+    indices.push(i);
+  }
+  return indices;
+}

package/backend/packages/tokenpatch/types.ts

@@ -0,0 +1,9 @@
+import type { Point } from 'web-tree-sitter';
+
+export interface Token {
+  text: string;
+  type: string;
+  startIndex: number; // byte offsets (tree-sitter)
+  endIndex: number;
+  startPosition: Point;
+}