pushwork 1.0.22 → 1.0.25

package/src/core/sync-engine.ts

@@ -30,7 +30,7 @@ import {
 	updateTextContent,
 	readDocContent,
 } from "../utils"
-import {isContentEqual} from "../utils/content"
+import {isContentEqual, contentHash} from "../utils/content"
 import {waitForSync, waitForBidirectionalSync} from "../utils/network-sync"
 import {SnapshotManager} from "./snapshot"
 import {ChangeDetector} from "./change-detection"
@@ -86,7 +86,8 @@ export class SyncEngine {
 		this.changeDetector = new ChangeDetector(
 			repo,
 			rootPath,
-			config.exclude_patterns
+			config.exclude_patterns,
+			config.artifact_directories || []
 		)
 		this.moveDetector = new MoveDetector(config.sync.move_detection_threshold)
 	}
@@ -248,6 +249,113 @@ export class SyncEngine {
 		}
 	}
 
+	/**
+	 * Recreate documents that failed to sync. Creates new Automerge documents
+	 * with the same content and updates all references (snapshot, parent directory).
+	 * Returns new handles that should be retried for sync.
+	 */
+	private async recreateFailedDocuments(
+		failedHandles: DocHandle<unknown>[],
+		snapshot: SyncSnapshot
+	): Promise<DocHandle<unknown>[]> {
+		const failedUrls = new Set(failedHandles.map(h => getPlainUrl(h.url)))
+		const newHandles: DocHandle<unknown>[] = []
+
+		// Find which paths correspond to the failed handles
+		for (const [filePath, entry] of snapshot.files.entries()) {
+			const plainUrl = getPlainUrl(entry.url)
+			if (!failedUrls.has(plainUrl)) continue
+
+			debug(`recreate: recreating document for ${filePath} (${plainUrl.slice(0, 20)}...)`)
+			out.taskLine(`Recreating document for ${filePath}`)
+
+			try {
+				// Read the current content from the old handle
+				const oldHandle = await this.repo.find<FileDocument>(plainUrl)
+				const doc = await oldHandle.doc()
+				if (!doc) {
+					debug(`recreate: could not read doc for ${filePath}, skipping`)
+					continue
+				}
+
+				const content = readDocContent(doc.content)
+				if (content === null) {
+					debug(`recreate: null content for ${filePath}, skipping`)
+					continue
+				}
+
+				// Create a fresh document
+				const fakeChange: DetectedChange = {
+					path: filePath,
+					changeType: ChangeType.LOCAL_ONLY,
+					fileType: this.isTextContent(content) ? FileType.TEXT : FileType.BINARY,
+					localContent: content,
+					remoteContent: null,
+				}
+				const newHandle = await this.createRemoteFile(fakeChange)
+				if (!newHandle) continue
+
+				const entryUrl = this.getEntryUrl(newHandle, filePath)
+
+				// Update snapshot entry
+				this.snapshotManager.updateFileEntry(snapshot, filePath, {
+					...entry,
+					url: entryUrl,
+					head: newHandle.heads(),
+					...(this.isArtifactPath(filePath) ? {contentHash: contentHash(content)} : {}),
+				})
+
+				// Update parent directory entry to point to new document
+				const pathParts = filePath.split("/")
+				const fileName = pathParts.pop() || ""
+				const dirPath = pathParts.join("/")
+
+				let dirUrl: AutomergeUrl
+				if (!dirPath || dirPath === "") {
+					dirUrl = snapshot.rootDirectoryUrl!
+				} else {
+					const dirEntry = snapshot.directories.get(dirPath)
+					if (!dirEntry) continue
+					dirUrl = dirEntry.url
+				}
+
+				const dirHandle = await this.repo.find<DirectoryDocument>(getPlainUrl(dirUrl))
+				dirHandle.change((d: DirectoryDocument) => {
+					const idx = d.docs.findIndex(
+						e => e.name === fileName && e.type === "file"
+					)
+					if (idx !== -1) {
+						d.docs[idx].url = entryUrl
+					}
+				})
+
+				// Track new handles
+				this.handlesByPath.set(filePath, newHandle)
+				this.handlesByPath.set(dirPath, dirHandle)
+				newHandles.push(newHandle)
+				newHandles.push(dirHandle)
+
+				debug(`recreate: created new doc for ${filePath} -> ${newHandle.url.slice(0, 20)}...`)
+			} catch (error) {
+				debug(`recreate: failed for ${filePath}: ${error}`)
+				out.taskLine(`Failed to recreate ${filePath}: ${error}`, true)
+			}
+		}
+
+		// Also check directory documents
+		for (const [dirPath, entry] of snapshot.directories.entries()) {
+			const plainUrl = getPlainUrl(entry.url)
+			if (!failedUrls.has(plainUrl)) continue
+
+			// Directory docs can't be easily recreated (they reference children).
+			// Just log a warning — the child recreation above should handle most cases.
+			debug(`recreate: directory ${dirPath || "(root)"} failed to sync, cannot recreate`)
+			out.taskLine(`Warning: directory ${dirPath || "(root)"} failed to sync`, true)
+		}
+
+		return newHandles
+	}
+
 	/**
 	 * Push local changes to server without pulling remote changes.
 	 * Detect changes, push to Automerge docs, upload to server. No bidirectional wait, no pull.
@@ -324,11 +432,30 @@ export class SyncEngine {
 						)
 						debug(`pushToRemote: waiting for ${allHandles.length} handles to sync to server`)
 						out.update(`Uploading ${allHandles.length} documents to sync server`)
-						await waitForSync(
+						const {failed} = await waitForSync(
 							allHandles,
 							this.config.sync_server_storage_id
 						)
-						debug("pushToRemote: all handles synced to server")
+
+						// Recreate failed documents and retry once
+						if (failed.length > 0) {
+							debug(`pushToRemote: ${failed.length} documents failed, recreating`)
+							out.update(`Recreating ${failed.length} failed documents`)
+							const retryHandles = await this.recreateFailedDocuments(failed, snapshot)
+							if (retryHandles.length > 0) {
+								debug(`pushToRemote: retrying ${retryHandles.length} recreated handles`)
+								out.update(`Retrying ${retryHandles.length} recreated documents`)
+								const retry = await waitForSync(
+									retryHandles,
+									this.config.sync_server_storage_id
+								)
+								if (retry.failed.length > 0) {
+									result.warnings.push(`${retry.failed.length} documents still failed after recreation`)
+								}
+							}
+						}
+
+						debug("pushToRemote: sync to server complete")
 					}
 				} catch (error) {
 					debug(`pushToRemote: network sync error: ${error}`)
@@ -445,10 +572,29 @@ export class SyncEngine {
 						const handlePaths = Array.from(this.handlesByPath.keys())
 						debug(`sync: waiting for ${allHandles.length} handles to sync to server: ${handlePaths.slice(0, 10).map(p => p || "(root)").join(", ")}${handlePaths.length > 10 ? ` ...and ${handlePaths.length - 10} more` : ""}`)
 						out.update(`Uploading ${allHandles.length} documents to sync server`)
-						await waitForSync(
+						const {failed} = await waitForSync(
 							allHandles,
 							this.config.sync_server_storage_id
 						)
+
+						// Recreate failed documents and retry once
+						if (failed.length > 0) {
+							debug(`sync: ${failed.length} documents failed, recreating`)
+							out.update(`Recreating ${failed.length} failed documents`)
+							const retryHandles = await this.recreateFailedDocuments(failed, snapshot)
+							if (retryHandles.length > 0) {
+								debug(`sync: retrying ${retryHandles.length} recreated handles`)
+								out.update(`Retrying ${retryHandles.length} recreated documents`)
+								const retry = await waitForSync(
+									retryHandles,
+									this.config.sync_server_storage_id
+								)
+								if (retry.failed.length > 0) {
+									result.warnings.push(`${retry.failed.length} documents still failed after recreation`)
+								}
+							}
+						}
+
 						debug("sync: all handles synced to server")
 					}
 
@@ -714,6 +860,9 @@ export class SyncEngine {
 									head: handle.heads(),
 									extension: getFileExtension(change.path),
 									mimeType: getEnhancedMimeType(change.path),
+									...(this.isArtifactPath(change.path) && change.localContent
+										? {contentHash: contentHash(change.localContent)}
+										: {}),
 								}
 							)
 							result.filesChanged++
@@ -936,42 +1085,66 @@ export class SyncEngine {
 
 		// 3) Update the FileDocument name and content to match new location/state
 		try {
-			// Use plain URL for mutable handle
-			const handle = await this.repo.find<FileDocument>(
-				getPlainUrl(fromEntry.url)
-			)
-			const heads = fromEntry.head
-
-			// Update both name and content (if content changed during move)
-			changeWithOptionalHeads(handle, heads, (doc: FileDocument) => {
-				doc.name = toFileName
-
-				// If new content is provided, update it (handles move + modification case)
-				if (move.newContent !== undefined) {
-					if (typeof move.newContent === "string") {
-						updateTextContent(doc, ["content"], move.newContent)
-					} else {
-						doc.content = move.newContent
-					}
+			let entryUrl: AutomergeUrl
+			let finalHeads: UrlHeads
+
+			if (this.isArtifactPath(move.toPath)) {
+				// Artifact files use RawString — no diffing needed, just create a fresh doc
+				const content = move.newContent !== undefined
+					? move.newContent
+					: readDocContent((await (await this.repo.find<FileDocument>(getPlainUrl(fromEntry.url))).doc())?.content)
+				const fakeChange: DetectedChange = {
+					path: move.toPath,
+					changeType: ChangeType.LOCAL_ONLY,
+					fileType: content != null && typeof content === "string" ? FileType.TEXT : FileType.BINARY,
+					localContent: content,
+					remoteContent: null,
 				}
-			})
+				const newHandle = await this.createRemoteFile(fakeChange)
+				if (!newHandle) return
+				entryUrl = this.getEntryUrl(newHandle, move.toPath)
+				finalHeads = newHandle.heads()
+			} else {
+				// Use plain URL for mutable handle
+				const handle = await this.repo.find<FileDocument>(
+					getPlainUrl(fromEntry.url)
+				)
+				const heads = fromEntry.head
+
+				// Update both name and content (if content changed during move)
+				changeWithOptionalHeads(handle, heads, (doc: FileDocument) => {
+					doc.name = toFileName
+
+					// If new content is provided, update it (handles move + modification case)
+					if (move.newContent !== undefined) {
+						if (typeof move.newContent === "string") {
+							updateTextContent(doc, ["content"], move.newContent)
+						} else {
+							doc.content = move.newContent
+						}
+					}
+				})
+
+				entryUrl = this.getEntryUrl(handle, move.toPath)
+				finalHeads = handle.heads()
 
-			// Get appropriate URL for directory entry
-			const entryUrl = this.getEntryUrl(handle, move.toPath)
+				// Track file handle for network sync
+				this.handlesByPath.set(move.toPath, handle)
+			}
 
 			// 4) Add file entry to destination directory
 			await this.addFileToDirectory(snapshot, move.toPath, entryUrl)
 
-			// Track file handle for network sync
-			this.handlesByPath.set(move.toPath, handle)
-
 			// 5) Update snapshot entries
 			this.snapshotManager.removeFileEntry(snapshot, move.fromPath)
 			this.snapshotManager.updateFileEntry(snapshot, move.toPath, {
 				...fromEntry,
 				path: joinAndNormalizePath(this.rootPath, move.toPath),
 				url: entryUrl,
-				head: handle.heads(),
+				head: finalHeads,
+				...(this.isArtifactPath(move.toPath) && move.newContent != null
+					? {contentHash: contentHash(move.newContent)}
+					: {}),
 			})
 		} catch (e) {
 			// Failed to update file name - file may have been deleted
@@ -1076,6 +1249,9 @@ export class SyncEngine {
 					head: newHandle.heads(),
 					extension: getFileExtension(filePath),
 					mimeType: getEnhancedMimeType(filePath),
+					...(this.isArtifactPath(filePath)
+						? {contentHash: contentHash(content)}
+						: {}),
 				})
 			}
 			return
@@ -1130,27 +1306,14 @@ export class SyncEngine {
 	 * Delete remote file document
 	 */
 	private async deleteRemoteFile(
-		url: AutomergeUrl,
-		snapshot?: SyncSnapshot,
-		filePath?: string
+		_url: AutomergeUrl,
+		_snapshot?: SyncSnapshot,
+		_filePath?: string
 	): Promise<void> {
-		// In Automerge, we don't actually delete documents
-		// They become orphaned and will be garbage collected
-		// For now, we just mark them as deleted by clearing content
-		// Use plain URL for mutable handle
-		const handle = await this.repo.find<FileDocument>(getPlainUrl(url))
-		// const doc = await handle.doc(); // no longer needed
-		let heads
-		if (snapshot && filePath) {
-			heads = snapshot.files.get(filePath)?.head
-		}
-		changeWithOptionalHeads(handle, heads, (doc: FileDocument) => {
-			if (doc.content instanceof Uint8Array) {
-				doc.content = new Uint8Array(0)
-			} else {
-				updateTextContent(doc, ["content"], "")
-			}
-		})
+		// In Automerge, we don't actually delete documents.
+		// The file entry is removed from its parent directory, making the
+		// document orphaned. Clearing content via splice is expensive for
+		// large text files (every character is a CRDT op), so we skip it.
 	}
 
 	/**

package/src/types/snapshot.ts

@@ -9,6 +9,7 @@ export interface SnapshotFileEntry {
   head: UrlHeads; // Document head at last sync
   extension: string; // File extension
   mimeType: string; // MIME type
+  contentHash?: string; // SHA-256 of content at last sync (used by artifact files to skip remote reads)
 }
 
 /**

package/src/utils/content.ts

@@ -1,3 +1,13 @@
+import { createHash } from "crypto";
+
+/**
+ * Compute a SHA-256 hash of file content.
+ * Used to detect local changes for artifact files without reading remote docs.
+ */
+export function contentHash(content: string | Uint8Array): string {
+  return createHash("sha256").update(content).digest("hex");
+}
+
 /**
  * Compare two content pieces for equality
  */

package/src/utils/network-sync.ts

@@ -61,12 +61,13 @@ export async function waitForBidirectionalSync(
       ? getHandleHeads(handles)
       : await getAllDocumentHeads(repo, rootDirectoryUrl);
 
-    // Scale timeout proportionally to tree size after first scan
+    // After first scan: scale timeout to tree size and reset the clock.
+    // The first scan is just establishing a baseline — its duration
+    // shouldn't count against the stability-wait timeout.
     if (pollCount === 1) {
-      dynamicTimeoutMs = Math.max(timeoutMs, 5000 + currentHeads.size * 50);
-      if (dynamicTimeoutMs !== timeoutMs) {
-        debug(`waitForBidirectionalSync: scaled timeout to ${dynamicTimeoutMs}ms for ${currentHeads.size} docs`);
-      }
+      const scanDuration = Date.now() - startTime;
+      dynamicTimeoutMs = Math.max(timeoutMs, 5000 + currentHeads.size * 50) + scanDuration;
+      debug(`waitForBidirectionalSync: first scan took ${scanDuration}ms, timeout now ${dynamicTimeoutMs}ms for ${currentHeads.size} docs`);
     }
 
     // Check if heads are stable (no changes since last check)
@@ -206,126 +207,166 @@ function headsMapEqual(
 }
 
 /**
- * Wait for documents to sync to the remote server
+ * Result of waitForSync — lists which handles failed to sync.
+ */
+export interface SyncWaitResult {
+  failed: DocHandle<unknown>[];
+}
+
+/** Maximum documents to sync concurrently to avoid flooding the server */
+const SYNC_BATCH_SIZE = 10;
+
+/**
+ * Wait for a single document handle to sync to the server.
+ * Resolves with the handle on success, rejects with the handle on timeout.
+ */
+function waitForHandleSync(
+  handle: DocHandle<unknown>,
+  syncServerStorageId: StorageId,
+  timeoutMs: number,
+  startTime: number,
+): Promise<DocHandle<unknown>> {
+  return new Promise<DocHandle<unknown>>((resolve, reject) => {
+    let pollInterval: NodeJS.Timeout;
+
+    const cleanup = () => {
+      clearTimeout(timeout);
+      clearInterval(pollInterval);
+      handle.off("remote-heads", onRemoteHeads);
+    };
+
+    const onConverged = () => {
+      debug(`waitForSync: ${handle.url.slice(0, 20)}... converged in ${Date.now() - startTime}ms`);
+      cleanup();
+      resolve(handle);
+    };
+
+    const timeout = setTimeout(() => {
+      debug(`waitForSync: ${handle.url.slice(0, 20)}... timed out after ${timeoutMs}ms`);
+      cleanup();
+      reject(handle);
+    }, timeoutMs);
+
+    const isConverged = () => {
+      const localHeads = handle.heads();
+      const info = handle.getSyncInfo(syncServerStorageId);
+      return A.equals(localHeads, info?.lastHeads);
+    };
+
+    const onRemoteHeads = ({
+      storageId,
+    }: {
+      storageId: StorageId;
+      heads: any;
+    }) => {
+      if (storageId === syncServerStorageId && isConverged()) {
+        onConverged();
+      }
+    };
+
+    // Initial check
+    if (isConverged()) {
+      cleanup();
+      resolve(handle);
+      return;
+    }
+
+    // Start polling and event listening
+    pollInterval = setInterval(() => {
+      if (isConverged()) {
+        onConverged();
+      }
+    }, 100);
+
+    handle.on("remote-heads", onRemoteHeads);
+  });
+}
+
+/**
+ * Wait for documents to sync to the remote server.
+ * Processes handles in batches to avoid flooding the server.
+ * Returns a result with any failed handles instead of throwing,
+ * so callers can attempt recovery (e.g. recreating documents).
  */
 export async function waitForSync(
   handlesToWaitOn: DocHandle<unknown>[],
   syncServerStorageId?: StorageId,
   timeoutMs: number = 60000,
-): Promise<void> {
+): Promise<SyncWaitResult> {
   const startTime = Date.now();
 
   if (!syncServerStorageId) {
     debug("waitForSync: no sync server storage ID, skipping");
-    return;
+    return { failed: [] };
   }
 
   if (handlesToWaitOn.length === 0) {
     debug("waitForSync: no documents to sync");
-    return;
+    return { failed: [] };
   }
 
-  debug(`waitForSync: waiting for ${handlesToWaitOn.length} documents (timeout=${timeoutMs}ms)`);
+  debug(`waitForSync: waiting for ${handlesToWaitOn.length} documents (timeout=${timeoutMs}ms, batchSize=${SYNC_BATCH_SIZE})`);
 
+  // Separate already-synced from needs-sync
+  const needsSync: DocHandle<unknown>[] = [];
   let alreadySynced = 0;
 
-  const promises = handlesToWaitOn.map((handle) => {
-    // Check if already synced
+  for (const handle of handlesToWaitOn) {
     const heads = handle.heads();
     const syncInfo = handle.getSyncInfo(syncServerStorageId);
     const remoteHeads = syncInfo?.lastHeads;
-    const wasAlreadySynced = A.equals(heads, remoteHeads);
-
-    if (wasAlreadySynced) {
+    if (A.equals(heads, remoteHeads)) {
       alreadySynced++;
       debug(`waitForSync: ${handle.url.slice(0, 20)}... already synced`);
-      return Promise.resolve();
+    } else {
+      debug(`waitForSync: ${handle.url.slice(0, 20)}... needs sync (remoteHeads=${remoteHeads ? 'present' : 'missing'})`);
+      needsSync.push(handle);
     }
+  }
 
-    debug(`waitForSync: ${handle.url.slice(0, 20)}... waiting for convergence (remoteHeads=${remoteHeads ? 'present' : 'missing'})`);
-
-    // Wait for convergence
-    return new Promise<void>((resolve, reject) => {
-      let pollInterval: NodeJS.Timeout;
-
-      const cleanup = () => {
-        clearTimeout(timeout);
-        clearInterval(pollInterval);
-        handle.off("remote-heads", onRemoteHeads);
-      };
-
-      const onConverged = () => {
-        debug(`waitForSync: ${handle.url.slice(0, 20)}... converged in ${Date.now() - startTime}ms`);
-        cleanup();
-        resolve();
-      };
-
-      const timeout = setTimeout(() => {
-        debug(`waitForSync: ${handle.url.slice(0, 20)}... timed out after ${timeoutMs}ms`);
-        cleanup();
-        reject(
-          new Error(
-            `Sync timeout after ${timeoutMs}ms for document ${handle.url}`,
-          ),
-        );
-      }, timeoutMs);
-
-      const isConverged = () => {
-        const localHeads = handle.heads();
-        const info = handle.getSyncInfo(syncServerStorageId);
-        return A.equals(localHeads, info?.lastHeads);
-      };
-
-      const onRemoteHeads = ({
-        storageId,
-      }: {
-        storageId: StorageId;
-        heads: any;
-      }) => {
-        if (storageId === syncServerStorageId && isConverged()) {
-          onConverged();
-        }
-      };
+  if (needsSync.length > 0) {
+    debug(`waitForSync: ${alreadySynced} already synced, ${needsSync.length} need sync`);
+    out.taskLine(`Uploading: ${alreadySynced}/${handlesToWaitOn.length} already synced, waiting for ${needsSync.length} more`);
+  } else {
+    debug(`waitForSync: all ${handlesToWaitOn.length} already synced`);
+    return { failed: [] };
+  }
 
-      const poll = () => {
-        if (isConverged()) {
-          onConverged();
-          return true;
-        }
-        return false;
-      };
+  // Process in batches to avoid flooding the server
+  const failed: DocHandle<unknown>[] = [];
+  let synced = alreadySynced;
 
-      // Initial check
-      if (poll()) {
-        return;
-      }
+  for (let i = 0; i < needsSync.length; i += SYNC_BATCH_SIZE) {
+    const batch = needsSync.slice(i, i + SYNC_BATCH_SIZE);
+    const batchNum = Math.floor(i / SYNC_BATCH_SIZE) + 1;
+    const totalBatches = Math.ceil(needsSync.length / SYNC_BATCH_SIZE);
 
-      // Start polling and event listening
-      pollInterval = setInterval(() => {
-        poll();
-      }, 100);
+    if (totalBatches > 1) {
+      debug(`waitForSync: batch ${batchNum}/${totalBatches} (${batch.length} docs)`);
+      out.update(`Uploading batch ${batchNum}/${totalBatches} (${synced}/${handlesToWaitOn.length} done)`);
+    }
 
-      handle.on("remote-heads", onRemoteHeads);
-    });
-  });
+    const results = await Promise.allSettled(
+      batch.map(handle => waitForHandleSync(handle, syncServerStorageId, timeoutMs, startTime))
+    );
 
-  const needSync = handlesToWaitOn.length - alreadySynced;
-  if (needSync > 0) {
-    debug(`waitForSync: ${alreadySynced} already synced, waiting for ${needSync} remaining`);
-    out.taskLine(`Uploading: ${alreadySynced}/${handlesToWaitOn.length} already synced, waiting for ${needSync} more`);
-  } else {
-    debug(`waitForSync: all ${handlesToWaitOn.length} already synced`);
+    for (const result of results) {
+      if (result.status === "rejected") {
+        failed.push(result.reason as DocHandle<unknown>);
+      } else {
+        synced++;
+      }
+    }
   }
 
-  try {
-    await Promise.all(promises);
-    const elapsed = Date.now() - startTime;
+  const elapsed = Date.now() - startTime;
+  if (failed.length > 0) {
+    debug(`waitForSync: ${failed.length} documents failed after ${elapsed}ms`);
+    out.taskLine(`Upload: ${synced} synced, ${failed.length} failed after ${(elapsed / 1000).toFixed(1)}s`, true);
+  } else {
     debug(`waitForSync: all ${handlesToWaitOn.length} documents synced in ${elapsed}ms (${alreadySynced} were already synced)`);
     out.taskLine(`All ${handlesToWaitOn.length} documents uploaded to server (${(elapsed / 1000).toFixed(1)}s)`);
-  } catch (error) {
-    const elapsed = Date.now() - startTime;
-    debug(`waitForSync: failed after ${elapsed}ms: ${error}`);
-    out.taskLine(`Upload to server failed after ${(elapsed / 1000).toFixed(1)}s: ${error}`, true);
-    throw error;
   }
+
+  return { failed };
 }

package/src/utils/repo-factory.ts

@@ -22,7 +22,6 @@ export async function createRepo(
       config.sync_server
     );
     repoConfig.network = [networkAdapter];
-    repoConfig.enableRemoteHeadsGossiping = true;
   }
 
   const repo = new Repo(repoConfig);