repoburg 1.3.11 → 1.3.13

package/backend/packages/tokenpatch/strategies/tree-sitter-tokenizer.ts

@@ -0,0 +1,37 @@
+import { Token } from '../types';
+import { TokenizerStrategy } from '../tokenizer.interface';
+import type { Tree, Node, Parser } from 'web-tree-sitter';
+
+export class TreeSitterTokenizer implements TokenizerStrategy {
+  constructor(private parser: Parser) {}
+
+  tokenize(content: string): Token[] {
+    const tree = this.parser.parse(content);
+    return this.collectTokens(tree, content);
+  }
+
+  private collectTokens(tree: Tree, code: string): Token[] {
+    const tokens: Token[] = [];
+
+    function visit(node: Node) {
+      if (node.childCount === 0) {
+        tokens.push({
+          text: code.slice(node.startIndex, node.endIndex),
+          type: node.type,
+          startIndex: node.startIndex,
+          endIndex: node.endIndex,
+          startPosition: node.startPosition,
+        });
+        return;
+      }
+      // By iterating over all children (not just named), we include punctuation
+      for (let i = 0; i < node.childCount; i++) {
+        const child = node.child(i);
+        if (child) visit(child);
+      }
+    }
+
+    visit(tree.rootNode);
+    return tokens;
+  }
+}

package/backend/packages/tokenpatch/tokenizer.interface.ts

@@ -0,0 +1,5 @@
+import { Token } from './types';
+
+export interface TokenizerStrategy {
+  tokenize(content: string): Token[];
+}

package/backend/packages/tokenpatch/tokens.ts

@@ -1,33 +1,15 @@
-import type { Tree, Node } from 'web-tree-sitter';
 import type { Token } from './types';
 
-export function collectTokens(tree: Tree, code: string): Token[] {
-  const tokens: Token[] = [];
-
-  function visit(node: Node) {
-    if (node.childCount === 0) {
-      tokens.push({
-        text: code.slice(node.startIndex, node.endIndex),
-        type: node.type,
-        startIndex: node.startIndex,
-        endIndex: node.endIndex,
-        startPosition: node.startPosition,
-      });
-      return;
-    }
-    // By iterating over all children (not just named), we include punctuation
-    // which is critical for anchor-based matching.
-    for (let i = 0; i < node.childCount; i++) {
-      const child = node.child(i);
-      if (child) visit(child);
-    }
-  }
-
-  visit(tree.rootNode);
-  return tokens;
-}
-
 export function tokensEqual(a: Token, b: Token): boolean {
+  // If using Tiktoken (BPE), we relax the comparison to ignore whitespace differences
+  // and newlines by trimming and stripping them. This helps when the patch might
+  // have slightly different indentation or spacing than the source.
+  if (a.type === 'bpe' || b.type === 'bpe') {
+    return (
+      a.text.replace(/\r?\n/g, '').trim() ===
+      b.text.replace(/\r?\n/g, '').trim()
+    );
+  }
   return a.text === b.text;
 }
 
@@ -47,4 +29,4 @@ export function findAllSequences(haystack: Token[], needle: Token[]): number[] {
     indices.push(i);
   }
   return indices;
-}
+}

package/backend/packages/tokenpatch/types.ts

@@ -2,8 +2,8 @@ import type { Point } from 'web-tree-sitter';
 
 export interface Token {
   text: string;
-  type: string;
-  startIndex: number; // byte offsets (tree-sitter)
+  type?: string;
+  startIndex: number; // byte offsets
   endIndex: number;
-  startPosition: Point;
-}
+  startPosition?: Point;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "repoburg",
-  "version": "1.3.11",
+  "version": "1.3.13",
   "description": "A local AI-powered software developer assistant that runs on your own machine.",
   "author": "Celal Ertug",
   "license": "SEE LICENSE IN LICENSE",
@@ -69,6 +69,7 @@
     "gpt-tokenizer": "^3.0.1",
     "http-proxy-middleware": "^3.0.0",
     "jose": "^5.6.3",
+    "js-tiktoken": "^1.0.21",
     "libphonenumber-js": "^1.12.10",
     "mermaid": "^11.10.1",
     "open": "^10.1.2",