pushwork 1.0.0 → 1.0.3

package/src/cli.ts

@@ -12,6 +12,7 @@ import {
   checkout,
   commit,
   url,
+  debug,
 } from "./cli/commands";
 
 /**
@@ -262,6 +263,34 @@ Note: This command outputs only the URL, making it useful for scripts.`
     })
   );
 
+// Debug command
+program
+  .command("debug")
+  .description("Show internal debug information including lastSyncAt timestamp")
+  .argument("[path]", "Directory path", ".")
+  .option(
+    "-v, --verbose",
+    "Show verbose debug information including full document contents"
+  )
+  .addHelpText(
+    "after",
+    `
+Examples:
+  pushwork debug           # Show debug info for current directory
+  pushwork debug --verbose # Show verbose debug info including full document contents
+  pushwork debug ./repo    # Show debug info for specific directory
+  
+This command displays internal document state, including the lastSyncAt timestamp
+that gets updated when sync operations make changes.`
+  )
+  .action(
+    withErrorHandling(async (path: string, options) => {
+      await debug(path, {
+        verbose: options.verbose || false,
+      });
+    })
+  );
+
 // Global error handler
 process.on("unhandledRejection", (reason, promise) => {
   console.error(
@@ -274,6 +303,17 @@ process.on("unhandledRejection", (reason, promise) => {
 });
 
 process.on("uncaughtException", (error) => {
+  // Ignore WebSocket errors during shutdown - they're non-critical
+  const errorMessage = error instanceof Error ? error.message : String(error);
+  if (
+    errorMessage.includes("WebSocket") ||
+    errorMessage.includes("connection was established") ||
+    errorMessage.includes("was closed")
+  ) {
+    // Silently ignore WebSocket shutdown errors
+    return;
+  }
+
   console.error(chalk.red("Uncaught Exception:"), error);
   process.exit(1);
 });

package/src/core/change-detection.ts

@@ -268,27 +268,40 @@ export class ChangeDetector {
           if (!existingEntry) {
             // This is a remote file not in our snapshot
             const localContent = await this.getLocalContent(entryPath);
+            const remoteContent = await this.getCurrentRemoteContent(entry.url);
+            const remoteHead = await this.getCurrentRemoteHead(entry.url);
 
-            // Only create changes for files that exist locally
-            // Files that don't exist locally AND aren't in snapshot should be ignored
-            // (they were likely deleted and directory documents haven't been cleaned up yet)
-            if (localContent) {
-              // File exists locally but not in snapshot - this is a new local file
-              const remoteContent = await this.getCurrentRemoteContent(
-                entry.url
-              );
-
+            if (localContent && remoteContent) {
+              // File exists both locally and remotely but not in snapshot
               changes.push({
                 path: entryPath,
                 changeType: ChangeType.BOTH_CHANGED,
                 fileType: await this.getFileTypeFromContent(remoteContent),
                 localContent,
                 remoteContent,
-                remoteHead: await this.getCurrentRemoteHead(entry.url),
+                remoteHead,
+              });
+            } else if (localContent !== null && remoteContent === null) {
+              // File exists locally but not remotely (shouldn't happen in this flow)
+              changes.push({
+                path: entryPath,
+                changeType: ChangeType.LOCAL_ONLY,
+                fileType: await this.getFileTypeFromContent(localContent),
+                localContent,
+                remoteContent: null,
+              });
+            } else if (localContent === null && remoteContent !== null) {
+              // File exists remotely but not locally - this is what we need for clone!
+              changes.push({
+                path: entryPath,
+                changeType: ChangeType.REMOTE_ONLY,
+                fileType: await this.getFileTypeFromContent(remoteContent),
+                localContent: null,
+                remoteContent,
+                remoteHead,
               });
             }
-            // If file doesn't exist locally and isn't in snapshot, ignore it
-            // This prevents infinite sync loops with ghost entries from stale directory documents
+            // Only ignore if neither local nor remote content exists (ghost entry)
           }
         } else if (entry.type === "folder") {
           // Recursively process subdirectory

package/src/core/move-detection.ts

@@ -47,13 +47,17 @@ export class MoveDetector {
         deletedFile,
         snapshot
       );
-      if (!deletedContent) continue;
+      // CRITICAL: Check for null explicitly, not falsy values
+      // Empty strings "" are valid file content!
+      if (deletedContent === null) continue;
 
       let bestMatch: { file: DetectedChange; similarity: number } | null = null;
 
       for (const createdFile of createdFiles) {
         if (usedCreations.has(createdFile.path)) continue;
-        if (!createdFile.localContent) continue;
+        // CRITICAL: Check for null explicitly, not falsy values
+        // Empty strings "" are valid file content!
+        if (createdFile.localContent === null) continue;
 
         const similarity = await ContentSimilarity.calculateSimilarity(
           deletedContent,
@@ -78,6 +82,8 @@ export class MoveDetector {
           toPath: bestMatch.file.path,
           similarity: bestMatch.similarity,
           confidence,
+          // Capture new content (may include modifications)
+          newContent: bestMatch.file.localContent || undefined,
         });
 
         // Only consume the deletion/creation pair when we would auto-apply the move.

package/src/core/sync-engine.ts

@@ -1,3 +1,5 @@
+const myers = require("myers-diff");
+
 import {
   AutomergeUrl,
   Repo,
@@ -150,6 +152,13 @@ export class SyncEngine {
       result.errors.push(...commitResult.errors);
       result.warnings.push(...commitResult.warnings);
 
+      // Touch root directory if any changes were made
+      const hasChanges =
+        result.filesChanged > 0 || result.directoriesChanged > 0;
+      if (hasChanges) {
+        await this.touchRootDirectory(snapshot, dryRun);
+      }
+
       // Save updated snapshot if not dry run
       if (!dryRun) {
         await this.snapshotManager.save(snapshot);
@@ -176,6 +185,9 @@ export class SyncEngine {
    * Run full bidirectional sync
    */
   async sync(dryRun = false): Promise<SyncResult> {
+    const syncStartTime = Date.now();
+    const timings: { [key: string]: number } = {};
+
     const result: SyncResult = {
       success: false,
       filesChanged: 0,
@@ -189,37 +201,47 @@ export class SyncEngine {
 
     try {
       // Load current snapshot
+      const t0 = Date.now();
       let snapshot = await this.snapshotManager.load();
+      timings["load_snapshot"] = Date.now() - t0;
       if (!snapshot) {
         snapshot = this.snapshotManager.createEmpty();
       }
 
       // Backup snapshot before starting
+      const t1 = Date.now();
       if (!dryRun) {
         await this.snapshotManager.backup();
       }
+      timings["backup_snapshot"] = Date.now() - t1;
 
       // Detect all changes
+      const t2 = Date.now();
       const changes = await this.changeDetector.detectChanges(snapshot);
+      timings["detect_changes"] = Date.now() - t2;
 
       // Detect moves
+      const t3 = Date.now();
       const { moves, remainingChanges } = await this.moveDetector.detectMoves(
         changes,
         snapshot,
         this.rootPath
       );
+      timings["detect_moves"] = Date.now() - t3;
 
       if (changes.length > 0) {
         console.log(`🔄 Syncing ${changes.length} changes...`);
       }
 
       // Phase 1: Push local changes to remote
+      const t4 = Date.now();
       const phase1Result = await this.pushLocalChanges(
         remainingChanges,
         moves,
         snapshot,
         dryRun
       );
+      timings["phase1_push"] = Date.now() - t4;
 
       result.filesChanged += phase1Result.filesChanged;
       result.directoriesChanged += phase1Result.directoriesChanged;
@@ -228,6 +250,7 @@ export class SyncEngine {
 
       // Always wait for network sync when enabled (not just when local changes exist)
       // This is critical for clone scenarios where we need to pull remote changes
+      const t5 = Date.now();
       if (!dryRun && this.networkSyncEnabled) {
         try {
           // If we have a root directory URL, wait for it to sync
@@ -239,41 +262,136 @@ export class SyncEngine {
           }
 
           if (this.handlesToWaitOn.length > 0) {
+            const tWaitStart = Date.now();
             await waitForSync(
               this.handlesToWaitOn,
               getSyncServerStorageId(this.syncServerStorageId)
             );
+            timings["network_sync"] = Date.now() - tWaitStart;
+
+            // CRITICAL: Wait a bit after our changes reach the server to allow
+            // time for WebSocket to deliver OTHER peers' changes to us.
+            // waitForSync only ensures OUR changes reached the server, not that
+            // we've RECEIVED changes from other peers. This delay allows the
+            // WebSocket protocol to propagate peer changes before we re-detect.
+            // Without this, concurrent operations on different peers can miss
+            // each other due to timing races.
+            //
+            // Optimization: Only wait if we pushed changes (shorter delay if no changes)
+            const tDelayStart = Date.now();
+            const delayMs = phase1Result.filesChanged > 0 ? 200 : 100;
+            await new Promise((resolve) => setTimeout(resolve, delayMs));
+            timings["post_sync_delay"] = Date.now() - tDelayStart;
           }
         } catch (error) {
           console.error(`❌ Network sync failed: ${error}`);
           result.warnings.push(`Network sync failed: ${error}`);
         }
       }
+      timings["total_network"] = Date.now() - t5;
 
       // Re-detect remote changes after network sync to ensure fresh state
       // This fixes race conditions where we detect changes before server propagation
+      // NOTE: We DON'T update snapshot heads yet - that would prevent detecting remote changes!
+      const t6 = Date.now();
       const freshChanges = await this.changeDetector.detectChanges(snapshot);
       const freshRemoteChanges = freshChanges.filter(
         (c) =>
           c.changeType === ChangeType.REMOTE_ONLY ||
           c.changeType === ChangeType.BOTH_CHANGED
       );
+      timings["redetect_changes"] = Date.now() - t6;
 
       // Phase 2: Pull remote changes to local using fresh detection
+      const t7 = Date.now();
       const phase2Result = await this.pullRemoteChanges(
         freshRemoteChanges,
         snapshot,
         dryRun
       );
+      timings["phase2_pull"] = Date.now() - t7;
       result.filesChanged += phase2Result.filesChanged;
       result.directoriesChanged += phase2Result.directoriesChanged;
       result.errors.push(...phase2Result.errors);
       result.warnings.push(...phase2Result.warnings);
 
+      // CRITICAL FIX: Update snapshot heads AFTER pulling remote changes
+      // This ensures that change detection can find remote changes, and we only
+      // update the snapshot after the filesystem is in sync with the documents
+      const t8 = Date.now();
+      if (!dryRun) {
+        // Update file document heads
+        for (const [filePath, snapshotEntry] of snapshot.files.entries()) {
+          try {
+            const handle = await this.repo.find(snapshotEntry.url);
+            const currentHeads = handle.heads();
+            if (!A.equals(currentHeads, snapshotEntry.head)) {
+              // Update snapshot with current heads after pulling changes
+              snapshot.files.set(filePath, {
+                ...snapshotEntry,
+                head: currentHeads,
+              });
+            }
+          } catch (error) {
+            // Handle might not exist if file was deleted, skip
+            console.warn(`Could not update heads for ${filePath}: ${error}`);
+          }
+        }
+
+        // Update directory document heads
+        for (const [dirPath, snapshotEntry] of snapshot.directories.entries()) {
+          try {
+            const handle = await this.repo.find(snapshotEntry.url);
+            const currentHeads = handle.heads();
+            if (!A.equals(currentHeads, snapshotEntry.head)) {
+              // Update snapshot with current heads after pulling changes
+              snapshot.directories.set(dirPath, {
+                ...snapshotEntry,
+                head: currentHeads,
+              });
+            }
+          } catch (error) {
+            // Handle might not exist if directory was deleted, skip
+            console.warn(
+              `Could not update heads for directory ${dirPath}: ${error}`
+            );
+          }
+        }
+      }
+      timings["update_snapshot_heads"] = Date.now() - t8;
+
+      // Touch root directory if any changes were made during sync
+      const t9 = Date.now();
+      const hasChanges =
+        result.filesChanged > 0 || result.directoriesChanged > 0;
+      if (hasChanges) {
+        await this.touchRootDirectory(snapshot, dryRun);
+      }
+      timings["touch_root"] = Date.now() - t9;
+
       // Save updated snapshot if not dry run
+      const t10 = Date.now();
       if (!dryRun) {
         await this.snapshotManager.save(snapshot);
       }
+      timings["save_snapshot"] = Date.now() - t10;
+
+      // Output timing breakdown if enabled via environment variable
+      if (process.env.PUSHWORK_TIMING === "1") {
+        const totalTime = Date.now() - syncStartTime;
+        console.error("\n⏱️  Sync Timing Breakdown:");
+        for (const [key, ms] of Object.entries(timings)) {
+          const pct = ((ms / totalTime) * 100).toFixed(1);
+          console.error(
+            `  ${key.padEnd(25)} ${ms.toString().padStart(5)}ms (${pct}%)`
+          );
+        }
+        console.error(
+          `  ${"TOTAL".padEnd(25)} ${totalTime
+            .toString()
+            .padStart(5)}ms (100.0%)\n`
+        );
+      }
 
       result.success = result.errors.length === 0;
       return result;
@@ -407,7 +525,9 @@ export class SyncEngine {
   ): Promise<void> {
     const snapshotEntry = snapshot.files.get(change.path);
 
-    if (!change.localContent) {
+    // CRITICAL: Check for null explicitly, not falsy values
+    // Empty strings "" and empty Uint8Array are valid file content!
+    if (change.localContent === null) {
       // File was deleted locally
       if (snapshotEntry) {
         console.log(`🗑️  ${change.path}`);
@@ -438,6 +558,8 @@ export class SyncEngine {
           dryRun
         );
 
+        // CRITICAL FIX: Update snapshot with heads AFTER adding to directory
+        // The addFileToDirectory call above may have changed the document heads
         this.snapshotManager.updateFileEntry(snapshot, change.path, {
           path: normalizePath(this.rootPath + "/" + change.path),
           url: handle.url,
@@ -449,6 +571,32 @@ export class SyncEngine {
     } else {
       // Update existing file
       console.log(`📝 ${change.path}`);
+
+      // log the change in detail for debugging
+      // split out remotea nd local content so we don't overwhelm the logs
+      const { remoteContent, localContent, ...rest } = change;
+      console.log(`🔍 Change in detail:`, rest);
+
+      // compare the local and remote content and make a diff so we can
+      // see what happened between the two
+      const { diff, changed } = require("myers-diff");
+      const lhs = change.remoteContent ? change.remoteContent.toString() : "";
+      const rhs = change.localContent ? change.localContent.toString() : "";
+      const changes = diff(lhs, rhs, { compare: "chars" });
+
+      for (const change of changes) {
+        if (changed(change.lhs)) {
+          // deleted
+          const { pos, text, del, length } = change.lhs;
+          console.log(`🔍 Deleted:`, { pos, text, del, length });
+        }
+        if (changed(change.rhs)) {
+          // added
+          const { pos, text, add, length } = change.rhs;
+          console.log(`🔍 Added:`, { pos, text, add, length });
+        }
+      }
+
       await this.updateRemoteFile(
         snapshotEntry.url,
         change.localContent,
@@ -471,11 +619,13 @@ export class SyncEngine {
 
     if (!change.remoteHead) {
       throw new Error(
-        `No remote head found for remote change to${change.path}`
+        `No remote head found for remote change to ${change.path}`
       );
     }
 
-    if (!change.remoteContent) {
+    // CRITICAL: Check for null explicitly, not falsy values
+    // Empty strings "" and empty Uint8Array are valid file content!
+    if (change.remoteContent === null) {
       // File was deleted remotely
       console.log(`🗑️  ${change.path}`);
       if (!dryRun) {
@@ -568,17 +718,39 @@ export class SyncEngine {
         dryRun
       );
 
-      // 3) Update the FileDocument name to match new basename
+      // 3) Update the FileDocument name and content to match new location/state
       try {
         const handle = await this.repo.find<FileDocument>(fromEntry.url);
         const heads = fromEntry.head;
+
+        // Update both name and content (if content changed during move)
         if (heads && heads.length > 0) {
           handle.changeAt(heads, (doc: FileDocument) => {
             doc.name = toFileName;
+
+            // If new content is provided, update it (handles move + modification case)
+            if (move.newContent !== undefined) {
+              const isText = this.isTextContent(move.newContent);
+              if (isText && typeof move.newContent === "string") {
+                updateText(doc, ["content"], move.newContent);
+              } else {
+                doc.content = move.newContent;
+              }
+            }
           });
         } else {
           handle.change((doc: FileDocument) => {
             doc.name = toFileName;
+
+            // If new content is provided, update it (handles move + modification case)
+            if (move.newContent !== undefined) {
+              const isText = this.isTextContent(move.newContent);
+              if (isText && typeof move.newContent === "string") {
+                updateText(doc, ["content"], move.newContent);
+              } else {
+                doc.content = move.newContent;
+              }
+            }
           });
         }
         // Track file handle for network sync
@@ -606,7 +778,9 @@ export class SyncEngine {
     change: DetectedChange,
     dryRun: boolean
   ): Promise<DocHandle<FileDocument> | null> {
-    if (dryRun || !change.localContent) return null;
+    // CRITICAL: Check for null explicitly, not falsy values
+    // Empty strings "" and empty Uint8Array are valid file content!
+    if (dryRun || change.localContent === null) return null;
 
     const isText = this.isTextContent(change.localContent);
 
@@ -657,11 +831,26 @@ export class SyncEngine {
     const currentContent = doc?.content;
     const contentChanged = !isContentEqual(content, currentContent);
 
+    // CRITICAL FIX: Always update snapshot heads, even when content is identical
+    // This prevents stale head issues that cause false change detection
+    const snapshotEntry = snapshot.files.get(filePath);
+    if (snapshotEntry) {
+      // Update snapshot with current document heads
+      snapshot.files.set(filePath, {
+        ...snapshotEntry,
+        head: handle.heads(),
+      });
+    }
+
     if (!contentChanged) {
+      // Content is identical, but we've updated the snapshot heads above
+      // This prevents fresh change detection from seeing stale heads
+      console.log(
+        `🔍 Content is identical, but we've updated the snapshot heads above`
+      );
       return;
     }
 
-    const snapshotEntry = snapshot.files.get(filePath);
     const heads = snapshotEntry?.head;
 
     if (!heads) {
@@ -677,7 +866,8 @@ export class SyncEngine {
       }
     });
 
-    if (!dryRun) {
+    // Update snapshot with new heads after content change
+    if (!dryRun && snapshotEntry) {
       snapshot.files.set(filePath, {
         ...snapshotEntry,
         head: handle.heads(),
@@ -781,6 +971,12 @@ export class SyncEngine {
     }
     if (didChange) {
       this.handlesToWaitOn.push(dirHandle);
+
+      // CRITICAL FIX: Update snapshot with new directory heads immediately
+      // This prevents stale head issues that cause convergence problems
+      if (snapshotEntry) {
+        snapshotEntry.head = dirHandle.heads();
+      }
     }
   }
 
@@ -900,6 +1096,13 @@ export class SyncEngine {
       this.handlesToWaitOn.push(dirHandle);
       if (didChange) {
         this.handlesToWaitOn.push(parentHandle);
+
+        // CRITICAL FIX: Update parent directory heads in snapshot immediately
+        // This prevents stale head issues when parent directory is modified
+        const parentSnapshotEntry = snapshot.directories.get(parentPath);
+        if (parentSnapshotEntry) {
+          parentSnapshotEntry.head = parentHandle.heads();
+        }
       }
 
       // Update snapshot with new directory
@@ -950,6 +1153,8 @@ export class SyncEngine {
       this.handlesToWaitOn.push(dirHandle);
       const snapshotEntry = snapshot.directories.get(directoryPath);
       const heads = snapshotEntry?.head;
+      let didChange = false;
+
       if (heads) {
         dirHandle.changeAt(heads, (doc: DirectoryDocument) => {
           const indexToRemove = doc.docs.findIndex(
@@ -957,6 +1162,7 @@ export class SyncEngine {
           );
           if (indexToRemove !== -1) {
             doc.docs.splice(indexToRemove, 1);
+            didChange = true;
             console.log(
               `🗑️  Removed ${fileName} from directory ${
                 directoryPath || "root"
@@ -971,6 +1177,7 @@ export class SyncEngine {
           );
           if (indexToRemove !== -1) {
             doc.docs.splice(indexToRemove, 1);
+            didChange = true;
             console.log(
               `🗑️  Removed ${fileName} from directory ${
                 directoryPath || "root"
@@ -979,6 +1186,12 @@ export class SyncEngine {
           }
         });
       }
+
+      // CRITICAL FIX: Update snapshot with new directory heads immediately
+      // This prevents stale head issues that cause convergence problems
+      if (didChange && snapshotEntry) {
+        snapshotEntry.head = dirHandle.heads();
+      }
     } catch (error) {
       console.warn(
         `Failed to remove ${fileName} from directory ${
@@ -1164,4 +1377,54 @@ export class SyncEngine {
 
     return parts.join(", ");
   }
+
+  /**
+   * Update the lastSyncAt timestamp on the root directory document
+   */
+  private async touchRootDirectory(
+    snapshot: SyncSnapshot,
+    dryRun: boolean
+  ): Promise<void> {
+    if (dryRun || !snapshot.rootDirectoryUrl) {
+      return;
+    }
+
+    try {
+      const rootHandle = await this.repo.find<DirectoryDocument>(
+        snapshot.rootDirectoryUrl
+      );
+
+      const snapshotEntry = snapshot.directories.get("");
+      const heads = snapshotEntry?.head;
+
+      const timestamp = Date.now();
+
+      if (heads) {
+        rootHandle.changeAt(heads, (doc: DirectoryDocument) => {
+          doc.lastSyncAt = timestamp;
+        });
+      } else {
+        rootHandle.change((doc: DirectoryDocument) => {
+          doc.lastSyncAt = timestamp;
+        });
+      }
+
+      // Track root directory for network sync
+      this.handlesToWaitOn.push(rootHandle);
+
+      // CRITICAL FIX: Update root directory heads in snapshot immediately
+      // This prevents stale head issues when root directory is modified
+      if (snapshotEntry) {
+        snapshotEntry.head = rootHandle.heads();
+      }
+
+      console.log(
+        `🕒 Updated root directory lastSyncAt to ${new Date(
+          timestamp
+        ).toISOString()}`
+      );
+    } catch (error) {
+      console.warn(`Failed to update root directory lastSyncAt: ${error}`);
+    }
+  }
 }

package/src/types/documents.ts

@@ -15,6 +15,7 @@ export interface DirectoryEntry {
 export interface DirectoryDocument {
   "@patchwork": { type: "folder" };
   docs: DirectoryEntry[];
+  lastSyncAt?: number; // Timestamp of last sync operation that made changes
 }
 
 /**
@@ -69,4 +70,5 @@ export interface MoveCandidate {
   toPath: string;
   similarity: number;
   confidence: "auto" | "prompt" | "low";
+  newContent?: string | Uint8Array; // Content at destination (may differ from source if modified during move)
 }

package/src/utils/fs.ts

@@ -159,9 +159,13 @@ function isExcluded(
       }
     } else if (pattern.includes("*")) {
       // Glob pattern like "*.tmp"
-      const regex = new RegExp(
-        pattern.replace(/\*/g, ".*").replace(/\?/g, ".")
-      );
+      // CRITICAL FIX: Properly escape dots and anchor the pattern
+      // Convert glob to regex: *.tmp -> ^.*\.tmp$ (not /.*.tmp/ which matches "fuftmp.ts"!)
+      const regexPattern = pattern
+        .replace(/\./g, "\\.") // Escape dots first
+        .replace(/\*/g, ".*") // Then convert * to .*
+        .replace(/\?/g, "."); // And ? to single char
+      const regex = new RegExp(`^${regexPattern}$`); // Anchor to match full path
       if (regex.test(relativePath)) {
         return true;
       }