pushwork 1.0.3 → 1.0.5

package/src/core/change-detection.ts

@@ -171,6 +171,33 @@ export class ChangeDetector {
     const changes: DetectedChange[] = [];
 
     for (const [relativePath, snapshotEntry] of snapshot.files.entries()) {
+      // CRITICAL FIX: Check if file still exists in remote directory listing
+      // Files can be removed from the directory without their document heads changing
+      const stillExistsInDirectory = await this.fileExistsInRemoteDirectory(
+        snapshot.rootDirectoryUrl,
+        relativePath
+      );
+
+      if (!stillExistsInDirectory) {
+        // File was removed from remote directory listing
+        const localContent = await this.getLocalContent(relativePath);
+
+        // Only report as deleted if local file still exists
+        // (if local file is also deleted, detectLocalChanges handles it)
+        if (localContent !== null) {
+          changes.push({
+            path: relativePath,
+            changeType: ChangeType.REMOTE_ONLY,
+            fileType: FileType.TEXT,
+            localContent,
+            remoteContent: null, // File deleted remotely
+            localHead: snapshotEntry.head,
+            remoteHead: snapshotEntry.head,
+          });
+        }
+        continue;
+      }
+
       const currentRemoteHead = await this.getCurrentRemoteHead(
         snapshotEntry.url
       );
@@ -494,4 +521,72 @@ export class ChangeDetector {
       return ChangeType.BOTH_CHANGED;
     }
   }
+
+  /**
+   * Check if a file exists in the remote directory hierarchy
+   */
+  private async fileExistsInRemoteDirectory(
+    rootDirectoryUrl: AutomergeUrl | undefined,
+    filePath: string
+  ): Promise<boolean> {
+    if (!rootDirectoryUrl) return false;
+    const entry = await this.findFileInDirectoryHierarchy(
+      rootDirectoryUrl,
+      filePath
+    );
+    return entry !== null;
+  }
+
+  /**
+   * Find a file in the directory hierarchy by path
+   */
+  private async findFileInDirectoryHierarchy(
+    directoryUrl: AutomergeUrl,
+    filePath: string
+  ): Promise<{ name: string; type: string; url: AutomergeUrl } | null> {
+    try {
+      const pathParts = filePath.split("/");
+      let currentDirUrl = directoryUrl;
+
+      // Navigate through directories to find the parent directory
+      for (let i = 0; i < pathParts.length - 1; i++) {
+        const dirName = pathParts[i];
+        const dirHandle = await this.repo.find<DirectoryDocument>(
+          currentDirUrl
+        );
+        const dirDoc = await dirHandle.doc();
+
+        if (!dirDoc) return null;
+
+        const subDirEntry = dirDoc.docs.find(
+          (entry: { name: string; type: string; url: AutomergeUrl }) =>
+            entry.name === dirName && entry.type === "folder"
+        );
+
+        if (!subDirEntry) return null;
+        currentDirUrl = subDirEntry.url;
+      }
+
+      // Now look for the file in the final directory
+      const fileName = pathParts[pathParts.length - 1];
+      const finalDirHandle = await this.repo.find<DirectoryDocument>(
+        currentDirUrl
+      );
+      const finalDirDoc = await finalDirHandle.doc();
+
+      if (!finalDirDoc) return null;
+
+      const fileEntry = finalDirDoc.docs.find(
+        (entry: { name: string; type: string; url: AutomergeUrl }) =>
+          entry.name === fileName && entry.type === "file"
+      );
+
+      return fileEntry || null;
+    } catch (error) {
+      console.warn(
+        `Failed to find file ${filePath} in directory hierarchy: ${error}`
+      );
+      return null;
+    }
+  }
 }

package/src/core/move-detection.ts

@@ -1,18 +1,14 @@
-import {
-  SyncSnapshot,
-  MoveCandidate,
-  FileType,
-  SnapshotFileEntry,
-} from "../types";
-import { ContentSimilarity, readFileContent, getRelativePath } from "../utils";
+import { SyncSnapshot, MoveCandidate, SnapshotFileEntry } from "../types";
+import { calculateContentHash, isTextFile } from "../utils";
+import { stringSimilarity } from "../utils/string-similarity";
 import { DetectedChange, ChangeType } from "./change-detection";
 
 /**
- * Move detection engine
+ * Simplified move detection engine
  */
 export class MoveDetector {
-  private static readonly AUTO_THRESHOLD = 0.8;
-  private static readonly PROMPT_THRESHOLD = 0.5;
+  // Single threshold: either it's a move or it's not
+  private static readonly MOVE_THRESHOLD = 0.7;
 
   /**
    * Detect file moves by analyzing deleted and created files
@@ -22,7 +18,6 @@ export class MoveDetector {
     snapshot: SyncSnapshot,
     rootPath: string
   ): Promise<{ moves: MoveCandidate[]; remainingChanges: DetectedChange[] }> {
-    // Separate deletions and creations
     const deletedFiles = changes.filter(
       (c) => !c.localContent && c.changeType === ChangeType.LOCAL_ONLY
     );
@@ -43,28 +38,22 @@ export class MoveDetector {
 
     // Find potential moves by comparing content
     for (const deletedFile of deletedFiles) {
-      const deletedContent = await this.getDeletedFileContent(
-        deletedFile,
-        snapshot
-      );
-      // CRITICAL: Check for null explicitly, not falsy values
-      // Empty strings "" are valid file content!
+      const deletedContent = deletedFile.remoteContent;
       if (deletedContent === null) continue;
 
       let bestMatch: { file: DetectedChange; similarity: number } | null = null;
 
       for (const createdFile of createdFiles) {
         if (usedCreations.has(createdFile.path)) continue;
-        // CRITICAL: Check for null explicitly, not falsy values
-        // Empty strings "" are valid file content!
         if (createdFile.localContent === null) continue;
 
-        const similarity = await ContentSimilarity.calculateSimilarity(
+        const similarity = await this.calculateSimilarity(
           deletedContent,
-          createdFile.localContent
+          createdFile.localContent,
+          deletedFile.path
         );
 
-        if (similarity >= MoveDetector.PROMPT_THRESHOLD) {
+        if (similarity >= MoveDetector.MOVE_THRESHOLD) {
           if (!bestMatch || similarity > bestMatch.similarity) {
             bestMatch = { file: createdFile, similarity };
           }
@@ -72,31 +61,20 @@ export class MoveDetector {
       }
 
       if (bestMatch) {
-        const confidence = ContentSimilarity.getConfidenceLevel(
-          bestMatch.similarity
-        );
-
-        // Always report the potential move (for logging/prompting)
+        // If we detected a move above threshold, we apply it
         moves.push({
           fromPath: deletedFile.path,
           toPath: bestMatch.file.path,
           similarity: bestMatch.similarity,
-          confidence,
-          // Capture new content (may include modifications)
           newContent: bestMatch.file.localContent || undefined,
         });
 
-        // Only consume the deletion/creation pair when we would auto-apply the move.
-        // If we only want to prompt, leave the original changes in place so
-        // sync can still proceed as delete+create (avoids infinite warning loop).
-        if (bestMatch.similarity >= MoveDetector.AUTO_THRESHOLD) {
-          usedCreations.add(bestMatch.file.path);
-          usedDeletions.add(deletedFile.path);
-        }
+        // Consume the deletion and creation (move replaces both)
+        usedCreations.add(bestMatch.file.path);
+        usedDeletions.add(deletedFile.path);
       }
     }
 
-    // Filter out changes that are part of moves
     const remainingChanges = changes.filter(
       (change) =>
         !usedCreations.has(change.path) && !usedDeletions.has(change.path)
@@ -106,145 +84,77 @@ export class MoveDetector {
   }
 
   /**
-   * Get content of a deleted file from snapshot
-   */
-  private async getDeletedFileContent(
-    deletedFile: DetectedChange,
-    snapshot: SyncSnapshot
-  ): Promise<string | Uint8Array | null> {
-    const snapshotEntry = snapshot.files.get(deletedFile.path);
-    if (!snapshotEntry) return null;
-
-    // Return remote content if available, otherwise null
-    return deletedFile.remoteContent || null;
-  }
-
-  /**
-   * Group moves by confidence level
-   */
-  groupMovesByConfidence(moves: MoveCandidate[]): {
-    autoMoves: MoveCandidate[];
-    promptMoves: MoveCandidate[];
-    lowConfidenceMoves: MoveCandidate[];
-  } {
-    const autoMoves = moves.filter((m) => m.confidence === "auto");
-    const promptMoves = moves.filter((m) => m.confidence === "prompt");
-    const lowConfidenceMoves = moves.filter((m) => m.confidence === "low");
-
-    return { autoMoves, promptMoves, lowConfidenceMoves };
-  }
-
-  /**
-   * Validate move candidates to avoid conflicts
+   * Calculate similarity between two content pieces
+   * Optimized for speed while maintaining accuracy
    */
-  validateMoves(moves: MoveCandidate[]): {
-    validMoves: MoveCandidate[];
-    conflicts: Array<{ moves: MoveCandidate[]; reason: string }>;
-  } {
-    const validMoves: MoveCandidate[] = [];
-    const conflicts: Array<{ moves: MoveCandidate[]; reason: string }> = [];
-
-    // Check for multiple sources mapping to same destination
-    const destinationMap = new Map<string, MoveCandidate[]>();
-    for (const move of moves) {
-      if (!destinationMap.has(move.toPath)) {
-        destinationMap.set(move.toPath, []);
-      }
-      destinationMap.get(move.toPath)!.push(move);
+  private async calculateSimilarity(
+    content1: string | Uint8Array,
+    content2: string | Uint8Array,
+    path: string
+  ): Promise<number> {
+    if (content1 === content2) return 1.0;
+
+    // Early exit: size difference too large
+    const size1 =
+      typeof content1 === "string" ? content1.length : content1.length;
+    const size2 =
+      typeof content2 === "string" ? content2.length : content2.length;
+    const sizeDiff = Math.abs(size1 - size2) / Math.max(size1, size2);
+    if (sizeDiff > 0.5) return 0.0;
+
+    // Binary files: hash mismatch = not a move
+    const isText = await isTextFile(path);
+    if (!isText) return 0.0;
+
+    // Text files: use string similarity
+    const str1 =
+      typeof content1 === "string" ? content1 : this.bufferToString(content1);
+    const str2 =
+      typeof content2 === "string" ? content2 : this.bufferToString(content2);
+
+    // For small files (<4KB), compare full content
+    if (size1 < 4096 && size2 < 4096) {
+      return stringSimilarity(str1, str2);
     }
 
-    // Check for multiple destinations from same source
-    const sourceMap = new Map<string, MoveCandidate[]>();
-    for (const move of moves) {
-      if (!sourceMap.has(move.fromPath)) {
-        sourceMap.set(move.fromPath, []);
-      }
-      sourceMap.get(move.fromPath)!.push(move);
-    }
+    // For large files, sample 3 locations
+    const samples1 = this.getSamples(str1);
+    const samples2 = this.getSamples(str2);
 
-    for (const move of moves) {
-      const destinationConflicts = destinationMap.get(move.toPath)!;
-      const sourceConflicts = sourceMap.get(move.fromPath)!;
-
-      if (destinationConflicts.length > 1) {
-        conflicts.push({
-          moves: destinationConflicts,
-          reason: `Multiple files moving to ${move.toPath}`,
-        });
-      } else if (sourceConflicts.length > 1) {
-        conflicts.push({
-          moves: sourceConflicts,
-          reason: `File ${move.fromPath} has multiple potential destinations`,
-        });
-      } else {
-        validMoves.push(move);
-      }
+    let totalSimilarity = 0;
+    for (let i = 0; i < Math.min(samples1.length, samples2.length); i++) {
+      totalSimilarity += stringSimilarity(samples1[i], samples2[i]);
     }
 
-    return { validMoves, conflicts };
-  }
-
-  /**
-   * Apply move detection heuristics
-   */
-  applyHeuristics(moves: MoveCandidate[]): MoveCandidate[] {
-    return moves
-      .filter((move) => {
-        // Filter out moves within the same directory unless similarity is very high
-        const fromDir = this.getDirectoryPath(move.fromPath);
-        const toDir = this.getDirectoryPath(move.toPath);
-
-        if (fromDir === toDir && move.similarity < 0.9) {
-          return false;
-        }
-
-        // Filter out moves with very different file extensions unless similarity is perfect
-        const fromExt = this.getFileExtension(move.fromPath);
-        const toExt = this.getFileExtension(move.toPath);
-
-        if (fromExt !== toExt && move.similarity < 1.0) {
-          return false;
-        }
-
-        return true;
-      })
-      .sort((a, b) => b.similarity - a.similarity); // Sort by similarity descending
-  }
-
-  /**
-   * Get directory path from file path
-   */
-  private getDirectoryPath(filePath: string): string {
-    const lastSlash = filePath.lastIndexOf("/");
-    return lastSlash >= 0 ? filePath.substring(0, lastSlash) : "";
+    return totalSimilarity / Math.min(samples1.length, samples2.length);
   }
 
   /**
-   * Get file extension from file path
+   * Get representative samples from content (beginning, middle, end)
    */
-  private getFileExtension(filePath: string): string {
-    const lastDot = filePath.lastIndexOf(".");
-    const lastSlash = filePath.lastIndexOf("/");
+  private getSamples(str: string): string[] {
+    const CHUNK_SIZE = 1024;
+    const length = str.length;
 
-    if (lastDot > lastSlash && lastDot >= 0) {
-      return filePath.substring(lastDot + 1).toLowerCase();
+    if (length <= CHUNK_SIZE) {
+      return [str];
     }
 
-    return "";
-  }
-
-  /**
-   * Check if a move should be auto-applied
-   */
-  shouldAutoApply(move: MoveCandidate): boolean {
-    return move.confidence === "auto";
+    return [
+      str.slice(0, CHUNK_SIZE), // Beginning
+      str.slice(
+        Math.floor(length / 2) - Math.floor(CHUNK_SIZE / 2),
+        Math.floor(length / 2) + Math.floor(CHUNK_SIZE / 2)
+      ), // Middle
+      str.slice(-CHUNK_SIZE), // End
+    ];
   }
 
   /**
-   * Check if a move should prompt the user
+   * Convert buffer to string (for text comparison)
    */
-  shouldPromptUser(move: MoveCandidate): boolean {
-    return move.confidence === "prompt";
+  private bufferToString(buffer: Uint8Array): string {
+    return new TextDecoder().decode(buffer);
   }
 
   /**
@@ -254,22 +164,4 @@ export class MoveDetector {
     const percentage = Math.round(move.similarity * 100);
     return `${move.fromPath} → ${move.toPath} (${percentage}% similar)`;
   }
-
-  /**
-   * Calculate move statistics
-   */
-  calculateStats(moves: MoveCandidate[]): {
-    total: number;
-    auto: number;
-    prompt: number;
-    averageSimilarity: number;
-  } {
-    const total = moves.length;
-    const auto = moves.filter((m) => m.confidence === "auto").length;
-    const prompt = moves.filter((m) => m.confidence === "prompt").length;
-    const averageSimilarity =
-      total > 0 ? moves.reduce((sum, m) => sum + m.similarity, 0) / total : 0;
-
-    return { total, auto, prompt, averageSimilarity };
-  }
 }

package/src/core/sync-engine.ts

@@ -1,5 +1,3 @@
-const myers = require("myers-diff");
-
 import {
   AutomergeUrl,
   Repo,
@@ -89,8 +87,6 @@ export class SyncEngine {
    * Commit local changes only (no network sync)
    */
   async commitLocal(dryRun = false): Promise<SyncResult> {
-    console.log(`🚀 Starting local commit process (dryRun: ${dryRun})`);
-
     const result: SyncResult = {
       success: false,
       filesChanged: 0,
@@ -101,51 +97,33 @@ export class SyncEngine {
 
     try {
       // Load current snapshot
-      console.log(`📸 Loading current snapshot...`);
       let snapshot = await this.snapshotManager.load();
       if (!snapshot) {
-        console.log(`📸 No snapshot found, creating empty one`);
         snapshot = this.snapshotManager.createEmpty();
-      } else {
-        console.log(`📸 Snapshot loaded with ${snapshot.files.size} files`);
-        if (snapshot.rootDirectoryUrl) {
-          console.log(`🔗 Root directory URL: ${snapshot.rootDirectoryUrl}`);
-        }
       }
 
       // Backup snapshot before starting
       if (!dryRun) {
-        console.log(`💾 Backing up snapshot...`);
         await this.snapshotManager.backup();
       }
 
       // Detect all changes
-      console.log(`🔍 Detecting changes...`);
       const changes = await this.changeDetector.detectChanges(snapshot);
-      console.log(`🔍 Found ${changes.length} changes`);
 
       // Detect moves
-      console.log(`📦 Detecting moves...`);
       const { moves, remainingChanges } = await this.moveDetector.detectMoves(
         changes,
         snapshot,
         this.rootPath
       );
-      console.log(
-        `📦 Found ${moves.length} moves, ${remainingChanges.length} remaining changes`
-      );
 
       // Apply local changes only (no network sync)
-      console.log(`💾 Committing local changes...`);
       const commitResult = await this.pushLocalChanges(
         remainingChanges,
         moves,
         snapshot,
         dryRun
       );
-      console.log(
-        `💾 Commit complete: ${commitResult.filesChanged} files changed`
-      );
 
       result.filesChanged += commitResult.filesChanged;
       result.directoriesChanged += commitResult.directoriesChanged;
@@ -165,11 +143,9 @@ export class SyncEngine {
       }
 
       result.success = result.errors.length === 0;
-      console.log(`💾 Local commit ${result.success ? "completed" : "failed"}`);
 
       return result;
     } catch (error) {
-      console.error(`❌ Local commit failed: ${error}`);
       result.errors.push({
         path: this.rootPath,
         operation: "commitLocal",
@@ -194,6 +170,7 @@ export class SyncEngine {
       directoriesChanged: 0,
       errors: [],
       warnings: [],
+      timings: {},
     };
 
     // Reset handles to wait on
@@ -229,10 +206,6 @@ export class SyncEngine {
       );
       timings["detect_moves"] = Date.now() - t3;
 
-      if (changes.length > 0) {
-        console.log(`🔄 Syncing ${changes.length} changes...`);
-      }
-
       // Phase 1: Push local changes to remote
       const t4 = Date.now();
       const phase1Result = await this.pushLocalChanges(
@@ -251,6 +224,7 @@ export class SyncEngine {
       // Always wait for network sync when enabled (not just when local changes exist)
       // This is critical for clone scenarios where we need to pull remote changes
       const t5 = Date.now();
+      timings["documents_to_sync"] = this.handlesToWaitOn.length;
       if (!dryRun && this.networkSyncEnabled) {
         try {
           // If we have a root directory URL, wait for it to sync
@@ -376,24 +350,12 @@ export class SyncEngine {
       }
       timings["save_snapshot"] = Date.now() - t10;
 
-      // Output timing breakdown if enabled via environment variable
-      if (process.env.PUSHWORK_TIMING === "1") {
-        const totalTime = Date.now() - syncStartTime;
-        console.error("\n⏱️  Sync Timing Breakdown:");
-        for (const [key, ms] of Object.entries(timings)) {
-          const pct = ((ms / totalTime) * 100).toFixed(1);
-          console.error(
-            `  ${key.padEnd(25)} ${ms.toString().padStart(5)}ms (${pct}%)`
-          );
-        }
-        console.error(
-          `  ${"TOTAL".padEnd(25)} ${totalTime
-            .toString()
-            .padStart(5)}ms (100.0%)\n`
-        );
-      }
+      // Calculate total time
+      const totalTime = Date.now() - syncStartTime;
+      timings["total"] = totalTime;
 
       result.success = result.errors.length === 0;
+      result.timings = timings;
       return result;
     } catch (error) {
       result.errors.push({
@@ -423,28 +385,18 @@ export class SyncEngine {
       warnings: [],
     };
 
-    // Process moves first
+    // Process moves first - all detected moves are applied
     for (const move of moves) {
-      if (this.moveDetector.shouldAutoApply(move)) {
-        try {
-          await this.applyMoveToRemote(move, snapshot, dryRun);
-          result.filesChanged++;
-        } catch (error) {
-          result.errors.push({
-            path: move.fromPath,
-            operation: "move",
-            error: error as Error,
-            recoverable: true,
-          });
-        }
-      } else if (this.moveDetector.shouldPromptUser(move)) {
-        // Instead of creating a persistent loop, perform delete+create semantics
-        // so the working tree converges even without auto-apply.
-        result.warnings.push(
-          `Potential move detected: ${this.moveDetector.formatMove(
-            move
-          )} (${Math.round(move.similarity * 100)}% similar)`
-        );
+      try {
+        await this.applyMoveToRemote(move, snapshot, dryRun);
+        result.filesChanged++;
+      } catch (error) {
+        result.errors.push({
+          path: move.fromPath,
+          operation: "move",
+          error: error as Error,
+          recoverable: true,
+        });
       }
     }
 
@@ -530,7 +482,6 @@ export class SyncEngine {
     if (change.localContent === null) {
       // File was deleted locally
       if (snapshotEntry) {
-        console.log(`🗑️  ${change.path}`);
         await this.deleteRemoteFile(
           snapshotEntry.url,
           dryRun,
@@ -548,7 +499,6 @@ export class SyncEngine {
 
     if (!snapshotEntry) {
       // New file
-      console.log(`➕ ${change.path}`);
       const handle = await this.createRemoteFile(change, dryRun);
       if (!dryRun && handle) {
         await this.addFileToDirectory(
@@ -570,33 +520,6 @@ export class SyncEngine {
       }
     } else {
       // Update existing file
-      console.log(`📝 ${change.path}`);
-
-      // log the change in detail for debugging
-      // split out remotea nd local content so we don't overwhelm the logs
-      const { remoteContent, localContent, ...rest } = change;
-      console.log(`🔍 Change in detail:`, rest);
-
-      // compare the local and remote content and make a diff so we can
-      // see what happened between the two
-      const { diff, changed } = require("myers-diff");
-      const lhs = change.remoteContent ? change.remoteContent.toString() : "";
-      const rhs = change.localContent ? change.localContent.toString() : "";
-      const changes = diff(lhs, rhs, { compare: "chars" });
-
-      for (const change of changes) {
-        if (changed(change.lhs)) {
-          // deleted
-          const { pos, text, del, length } = change.lhs;
-          console.log(`🔍 Deleted:`, { pos, text, del, length });
-        }
-        if (changed(change.rhs)) {
-          // added
-          const { pos, text, add, length } = change.rhs;
-          console.log(`🔍 Added:`, { pos, text, add, length });
-        }
-      }
-
       await this.updateRemoteFile(
         snapshotEntry.url,
         change.localContent,
@@ -627,7 +550,6 @@ export class SyncEngine {
     // Empty strings "" and empty Uint8Array are valid file content!
     if (change.remoteContent === null) {
       // File was deleted remotely
-      console.log(`🗑️  ${change.path}`);
       if (!dryRun) {
         await removePath(localPath);
         this.snapshotManager.removeFileEntry(snapshot, change.path);
@@ -636,12 +558,6 @@ export class SyncEngine {
     }
 
     // Create or update local file
-    if (change.changeType === ChangeType.REMOTE_ONLY) {
-      console.log(`⬇️  ${change.path}`);
-    } else {
-      console.log(`🔀 ${change.path}`);
-    }
-
     if (!dryRun) {
       await writeFileContent(localPath, change.remoteContent);
 
@@ -931,10 +847,6 @@ export class SyncEngine {
       dryRun
     );
 
-    console.log(
-      `🔗 Adding ${fileName} (${fileUrl}) to directory ${parentDirUrl} (path: ${directoryPath})`
-    );
-
     const dirHandle = await this.repo.find<DirectoryDocument>(parentDirUrl);
 
     let didChange = false;