pushwork 1.1.6 → 1.2.2

package/src/core/sync-engine.ts

@@ -60,6 +60,64 @@ function changeWithOptionalHeads<T>(
 	}
 }
 
+/**
+ * Nuke an artifact directory's docs array and rebuild it from scratch.
+ * Entries must be spread into plain objects — pushing Automerge proxy objects
+ * back after splicing them out throws "Cannot create a reference to an
+ * existing document object".
+ */
+export function nukeAndRebuildDocs(
+	doc: DirectoryDocument,
+	dirPath: string,
+	newEntries: {name: string; url: AutomergeUrl}[],
+	updatedEntries: {name: string; url: AutomergeUrl}[],
+	deletedNames: string[],
+	subdirUpdates: {name: string; url: AutomergeUrl}[],
+): void {
+	const deletedSet = new Set(deletedNames)
+	const updatedMap = new Map(updatedEntries.map(e => [e.name, e.url]))
+	const newMap = new Map(newEntries.map(e => [e.name, e.url]))
+	const subdirMap = new Map(subdirUpdates.map(e => [e.name, e.url]))
+
+	const kept: DirectoryEntry[] = []
+	for (const entry of doc.docs) {
+		if (entry.type === "file" && deletedSet.has(entry.name)) {
+			out.taskLine(
+				`Removed ${entry.name} from ${
+					formatRelativePath(dirPath) || "root"
+				}`
+			)
+			continue
+		}
+		if (entry.type === "file" && updatedMap.has(entry.name)) {
+			kept.push({...entry, url: updatedMap.get(entry.name)!})
+			continue
+		}
+		if (entry.type === "file" && newMap.has(entry.name)) {
+			// Existing entry being re-added (e.g. from immutable string replacement)
+			kept.push({...entry, url: newMap.get(entry.name)!})
+			newMap.delete(entry.name)
+			continue
+		}
+		if (entry.type === "folder" && subdirMap.has(entry.name)) {
+			kept.push({...entry, url: subdirMap.get(entry.name)!})
+			continue
+		}
+		kept.push({...entry})
+	}
+
+	// Add genuinely new file entries
+	for (const [name, url] of newMap) {
+		kept.push({name, type: "file", url})
+	}
+
+	// Nuke and rebuild
+	doc.docs.splice(0, doc.docs.length)
+	for (const entry of kept) {
+		doc.docs.push(entry)
+	}
+}
+
 /**
  * Sync configuration constants
  */
@@ -139,16 +197,25 @@ export class SyncEngine {
 
 	/**
 	 * Get the appropriate URL for a subdirectory's directory entry.
-	 * Always uses plain URLs — versioned URLs on directories can cause
-	 * issues where consumers see a version without the docs array.
+	 * Artifact directories get versioned URLs (with heads) so consumers can
+	 * fetch the exact snapshotted version, matching how artifact files work.
+	 * Non-artifact directories get plain URLs for collaborative editing.
 	 */
-	private getDirEntryUrl(handle: DocHandle<unknown>): AutomergeUrl {
+	private getDirEntryUrl(handle: DocHandle<unknown>, dirPath: string): AutomergeUrl {
+		if (this.isArtifactPath(dirPath)) {
+			return this.getVersionedUrl(handle)
+		}
 		return getPlainUrl(handle.url)
 	}
 
 	/**
 	 * Set the root directory URL in the snapshot
 	 */
+	async getRootDirectoryUrl(): Promise<AutomergeUrl | undefined> {
+		const snapshot = await this.snapshotManager.load()
+		return snapshot?.rootDirectoryUrl
+	}
+
 	async setRootDirectoryUrl(url: AutomergeUrl): Promise<void> {
 		let snapshot = await this.snapshotManager.load()
 		if (!snapshot) {
@@ -365,7 +432,7 @@ export class SyncEngine {
 	/**
 	 * Run full bidirectional sync
 	 */
-	async sync(): Promise<SyncResult> {
+	async sync(options?: {sub?: boolean}): Promise<SyncResult> {
 		const result: SyncResult = {
 			success: false,
 			filesChanged: 0,
@@ -424,7 +491,6 @@ export class SyncEngine {
 					await waitForBidirectionalSync(
 						this.repo,
 						snapshot.rootDirectoryUrl,
-						this.config.sync_server_storage_id,
 						{
 							timeoutMs: 5000, // Increased timeout for initial sync
 							pollIntervalMs: 100,
@@ -468,6 +534,12 @@ export class SyncEngine {
 
 			// Wait for network sync (important for clone scenarios)
 			if (this.config.sync_enabled) {
+				const sub = options?.sub ?? false
+				// In Subduction mode, pass no StorageId so waitForSync
+				// falls back to head-stability polling. In WebSocket mode,
+				// pass the StorageId for precise getSyncInfo-based verification.
+				const storageId = sub ? undefined : this.config.sync_server_storage_id
+
 				try {
 					// Ensure root directory handle is tracked for sync
 					if (snapshot.rootDirectoryUrl) {
@@ -488,11 +560,13 @@ export class SyncEngine {
 						out.update(`Uploading ${allHandles.length} documents to sync server`)
 						const {failed} = await waitForSync(
 							allHandles,
-							this.config.sync_server_storage_id
+							storageId
 						)
 
-						// Recreate failed documents and retry once
-						if (failed.length > 0) {
+						// Recreate failed documents and retry once.
+						// Skip in Subduction mode — SubductionSource has its
+						// own heal-sync retry logic.
+						if (failed.length > 0 && !sub) {
 							debug(`sync: ${failed.length} documents failed, recreating`)
 							out.update(`Recreating ${failed.length} failed documents`)
 							const retryHandles = await this.recreateFailedDocuments(failed, snapshot)
@@ -501,7 +575,7 @@ export class SyncEngine {
 								out.update(`Retrying ${retryHandles.length} recreated documents`)
 								const retry = await waitForSync(
 									retryHandles,
-									this.config.sync_server_storage_id
+									storageId
 								)
 								if (retry.failed.length > 0) {
 									const msg = `${retry.failed.length} documents failed to sync to server after recreation`
@@ -514,6 +588,11 @@ export class SyncEngine {
 									})
 								}
 							}
+						} else if (failed.length > 0 && sub) {
+							const msg = `${failed.length} document${failed.length === 1 ? '' : 's'} did not converge during sync (Subduction will retry in the background; re-run sync to confirm)`
+							debug(`sync: ${msg}`)
+							out.taskLine(msg, true)
+							result.warnings.push(msg)
 						}
 
 						debug("sync: all handles synced to server")
@@ -527,7 +606,6 @@ export class SyncEngine {
 					await waitForBidirectionalSync(
 						this.repo,
 						snapshot.rootDirectoryUrl,
-						this.config.sync_server_storage_id,
 						{
 							timeoutMs: BIDIRECTIONAL_SYNC_TIMEOUT_MS,
 							pollIntervalMs: 100,
@@ -550,10 +628,15 @@ export class SyncEngine {
 							)
 						debug("sync: syncing root directory touch to server")
 						out.update("Syncing root directory update")
-						await waitForSync(
+						const rootSync = await waitForSync(
 							[rootHandle],
-							this.config.sync_server_storage_id
+							storageId
 						)
+						if (rootSync.failed.length > 0) {
+							const msg = "Root directory update did not converge to server; consumers may not see recent changes until next sync"
+							debug(`sync: ${msg}`)
+							result.warnings.push(msg)
+						}
 					}
 				} catch (error) {
 					debug(`sync: network sync error: ${error}`)
@@ -880,7 +963,7 @@ export class SyncEngine {
 							)
 						subdirUpdates.push({
 							name: childName,
-							url: this.getDirEntryUrl(childHandle),
+							url: this.getDirEntryUrl(childHandle, modifiedDir),
 						})
 					}
 				}
@@ -1380,7 +1463,7 @@ export class SyncEngine {
 						this.handlesByPath.set(directoryPath, childDirHandle)
 
 						// Get appropriate URL for directory entry
-						const entryUrl = this.getDirEntryUrl(childDirHandle)
+						const entryUrl = this.getDirEntryUrl(childDirHandle, directoryPath)
 
 						// Update snapshot with discovered directory
 						this.snapshotManager.updateDirectoryEntry(snapshot, directoryPath, {
@@ -1411,7 +1494,7 @@ export class SyncEngine {
 		const dirHandle = this.repo.create(dirDoc)
 
 		// Get appropriate URL for directory entry
-		const dirEntryUrl = this.getDirEntryUrl(dirHandle)
+		const dirEntryUrl = this.getDirEntryUrl(dirHandle, directoryPath)
 
 		// Add this directory to its parent
 		// Use plain URL for mutable handle
@@ -1555,53 +1638,10 @@ export class SyncEngine {
 		if (this.isArtifactPath(dirPath)) {
 			// Artifact directories are always nuked: rebuild docs array from scratch
 			// using a plain change() to avoid changeAt forking from stale heads.
-			const deletedSet = new Set(deletedNames)
-			const updatedMap = new Map(updatedEntries.map(e => [e.name, e.url]))
-			const newMap = new Map(newEntries.map(e => [e.name, e.url]))
-			const subdirMap = new Map(subdirUpdates.map(e => [e.name, e.url]))
-
 			dirHandle.change((doc: DirectoryDocument) => {
 				if (!doc.name) doc.name = dirName
 				if (!doc.title) doc.title = dirName
-
-				// Collect desired entries from current state + changes
-				const kept: DirectoryEntry[] = []
-				for (const entry of doc.docs) {
-					if (entry.type === "file" && deletedSet.has(entry.name)) {
-						out.taskLine(
-							`Removed ${entry.name} from ${
-								formatRelativePath(dirPath) || "root"
-							}`
-						)
-						continue
-					}
-					if (entry.type === "file" && updatedMap.has(entry.name)) {
-						kept.push({...entry, url: updatedMap.get(entry.name)!})
-						continue
-					}
-					if (entry.type === "file" && newMap.has(entry.name)) {
-						// Existing entry being re-added (e.g. from immutable string replacement)
-						kept.push({...entry, url: newMap.get(entry.name)!})
-						newMap.delete(entry.name)
-						continue
-					}
-					if (entry.type === "folder" && subdirMap.has(entry.name)) {
-						kept.push({...entry, url: subdirMap.get(entry.name)!})
-						continue
-					}
-					kept.push(entry)
-				}
-
-				// Add genuinely new file entries
-				for (const [name, url] of newMap) {
-					kept.push({name, type: "file", url})
-				}
-
-				// Nuke and rebuild
-				doc.docs.splice(0, doc.docs.length)
-				for (const entry of kept) {
-					doc.docs.push(entry)
-				}
+				nukeAndRebuildDocs(doc, dirPath, newEntries, updatedEntries, deletedNames, subdirUpdates)
 			})
 		} else {
 			changeWithOptionalHeads(dirHandle, heads, (doc: DirectoryDocument) => {

package/src/types/config.ts

@@ -6,6 +6,7 @@ import { StorageId } from "@automerge/automerge-repo";
 export const DEFAULT_SYNC_SERVER = "wss://sync3.automerge.org";
 export const DEFAULT_SYNC_SERVER_STORAGE_ID =
   "3760df37-a4c6-4f66-9ecd-732039a9385d" as StorageId;
+export const DEFAULT_SUBDUCTION_SERVER = "wss://subduction.sync.inkandswitch.com";
 
 /**
  * Global configuration options
@@ -25,6 +26,7 @@ export interface GlobalConfig {
  */
 export interface DirectoryConfig extends GlobalConfig {
   root_directory_url?: string;
+  subduction?: boolean;
   sync_enabled: boolean;
 }
 
@@ -42,6 +44,7 @@ export interface CloneOptions extends CommandOptions {
   force?: boolean; // Overwrite existing directory
   syncServer?: string; // Custom sync server URL
   syncServerStorageId?: StorageId; // Custom sync server storage ID
+  sub?: boolean;
 }
 
 /**
@@ -83,6 +86,7 @@ export interface CheckoutOptions extends CommandOptions {
 export interface InitOptions extends CommandOptions {
   syncServer?: string;
   syncServerStorageId?: StorageId;
+  sub?: boolean;
 }
 
 /**

package/src/utils/content.ts

@@ -16,7 +16,7 @@ export function isContentEqual(
   content2: string | Uint8Array | null
 ): boolean {
   if (content1 === content2) return true;
-  if (!content1 || !content2) return false;
+  if (content1 == null || content2 == null) return false;
 
   if (typeof content1 !== typeof content2) return false;
 

package/src/utils/network-sync.ts

@@ -21,13 +21,11 @@ function debug(...args: any[]) {
  *
  * @param repo - The Automerge repository
  * @param rootDirectoryUrl - The root directory URL to start traversal from
- * @param syncServerStorageId - The sync server storage ID
  * @param options - Configuration options
  */
 export async function waitForBidirectionalSync(
   repo: Repo,
   rootDirectoryUrl: AutomergeUrl | undefined,
-  syncServerStorageId: StorageId | undefined,
   options: {
     timeoutMs?: number;
     pollIntervalMs?: number;
@@ -42,7 +40,7 @@ export async function waitForBidirectionalSync(
     handles,
   } = options;
 
-  if (!syncServerStorageId || !rootDirectoryUrl) {
+  if (!rootDirectoryUrl) {
     return;
   }
 
@@ -295,16 +293,20 @@ export async function waitForSync(
 ): Promise<SyncWaitResult> {
   const startTime = Date.now();
 
-  if (!syncServerStorageId) {
-    debug("waitForSync: no sync server storage ID, skipping");
-    return { failed: [] };
-  }
-
   if (handlesToWaitOn.length === 0) {
     debug("waitForSync: no documents to sync");
     return { failed: [] };
   }
 
+  // When no StorageId is available (Subduction mode), use head-stability
+  // polling. The SubductionSource handles sync internally — we just wait
+  // for each handle's heads to stop changing.
+  if (!syncServerStorageId) {
+    debug(`waitForSync: no storage ID, using head-stability polling for ${handlesToWaitOn.length} documents`);
+    out.taskLine(`Waiting for ${handlesToWaitOn.length} documents to sync`);
+    return waitForSyncViaHeadStability(handlesToWaitOn, timeoutMs, startTime);
+  }
+
   debug(`waitForSync: waiting for ${handlesToWaitOn.length} documents (timeout=${timeoutMs}ms, batchSize=${SYNC_BATCH_SIZE})`);
 
   // Separate already-synced from needs-sync
@@ -370,3 +372,85 @@ export async function waitForSync(
 
   return { failed };
 }
+
+/**
+ * Wait for sync by polling head stability (Subduction mode).
+ * Each handle's heads are polled until they remain unchanged for
+ * several consecutive checks, indicating the SubductionSource has
+ * finished syncing.
+ */
+async function waitForSyncViaHeadStability(
+  handles: DocHandle<unknown>[],
+  timeoutMs: number,
+  startTime: number,
+): Promise<SyncWaitResult> {
+  const failed: DocHandle<unknown>[] = [];
+  let synced = 0;
+
+  // Process in batches
+  for (let i = 0; i < handles.length; i += SYNC_BATCH_SIZE) {
+    const batch = handles.slice(i, i + SYNC_BATCH_SIZE);
+
+    const results = await Promise.allSettled(
+      batch.map(handle => waitForHandleHeadStability(handle, timeoutMs, startTime))
+    );
+
+    for (const result of results) {
+      if (result.status === "rejected") {
+        failed.push(result.reason as DocHandle<unknown>);
+      } else {
+        synced++;
+      }
+    }
+  }
+
+  const elapsed = Date.now() - startTime;
+  if (failed.length > 0) {
+    debug(`waitForSync(heads): ${failed.length} documents failed after ${elapsed}ms`);
+    out.taskLine(`Sync: ${synced} synced, ${failed.length} timed out after ${(elapsed / 1000).toFixed(1)}s`, true);
+  } else {
+    debug(`waitForSync(heads): all ${handles.length} documents synced in ${elapsed}ms`);
+    out.taskLine(`All ${handles.length} documents synced (${(elapsed / 1000).toFixed(1)}s)`);
+  }
+
+  return { failed };
+}
+
+/**
+ * Wait for a single handle's heads to stabilize.
+ * Polls heads at 100ms intervals; resolves after 3 consecutive stable
+ * checks, rejects on timeout.
+ */
+function waitForHandleHeadStability(
+  handle: DocHandle<unknown>,
+  timeoutMs: number,
+  startTime: number,
+): Promise<DocHandle<unknown>> {
+  return new Promise<DocHandle<unknown>>((resolve, reject) => {
+    let lastHeads = JSON.stringify(handle.heads());
+    let stableCount = 0;
+    const stableRequired = 3;
+
+    const pollInterval = setInterval(() => {
+      const currentHeads = JSON.stringify(handle.heads());
+      if (currentHeads === lastHeads) {
+        stableCount++;
+        if (stableCount >= stableRequired) {
+          clearInterval(pollInterval);
+          clearTimeout(timeout);
+          debug(`waitForSync(heads): ${handle.url}... converged in ${Date.now() - startTime}ms`);
+          resolve(handle);
+        }
+      } else {
+        stableCount = 0;
+        lastHeads = currentHeads;
+      }
+    }, 100);
+
+    const timeout = setTimeout(() => {
+      clearInterval(pollInterval);
+      debug(`waitForSync(heads): ${handle.url}... timed out after ${timeoutMs}ms`);
+      reject(handle);
+    }, timeoutMs);
+  });
+}

package/src/utils/output.ts

@@ -345,7 +345,7 @@ export class Output {
       this.taskOriginalMessage = null;
       this.taskCurrentMessage = null;
     }
-    console.log(
+    console.error(
       chalk.red(
         message instanceof Error
           ? message.message
@@ -367,7 +367,7 @@ export class Output {
       this.taskOriginalMessage = null;
       this.taskCurrentMessage = null;
     }
-    console.log(
+    console.error(
       `\n${chalk.bgRed.white(` ${label} `)}${message && ` ${message}`}`
     );
   }
@@ -400,19 +400,19 @@ export class Output {
 
     if (error instanceof Error) {
       // Error type and message
-      console.log(chalk.red(`${error.name}: ${error.message}`));
+      console.error(chalk.red(`${error.name}: ${error.message}`));
 
       // Stack trace
       if (error.stack) {
-        console.log("");
-        console.log(chalk.dim("Stack trace:"));
+        console.error("");
+        console.error(chalk.dim("Stack trace:"));
         const stackLines = error.stack.split("\n").slice(1); // Skip first line (error message)
         stackLines.forEach((line) =>
-          console.log(chalk.dim(`  ${line.trim()}`))
+          console.error(chalk.dim(`  ${line.trim()}`))
         );
       }
     } else {
-      console.log(chalk.red(String(error)));
+      console.error(chalk.red(String(error)));
     }
 
     process.exit(exitCode);

package/src/utils/repo-factory.ts

@@ -1,28 +1,142 @@
-import { Repo } from "@automerge/automerge-repo";
+import { type Repo, type RepoConfig, type NetworkAdapterInterface } from "@automerge/automerge-repo";
 import { NodeFSStorageAdapter } from "@automerge/automerge-repo-storage-nodefs";
-import { BrowserWebSocketClientAdapter } from "@automerge/automerge-repo-network-websocket";
+import * as fs from "fs/promises";
 import * as path from "path";
 import { DirectoryConfig } from "../types";
 
 /**
- * Create an Automerge repository with configuration-based setup
+ * Perform a real ESM dynamic import that tsc won't rewrite to require().
+ *
+ * TypeScript with `"module": "commonjs"` compiles `await import("x")` to
+ * `require("x")`, which resolves CJS entries instead of ESM entries. The
+ * Wasm module instance is different between the CJS and ESM module graphs,
+ * so initializing via CJS require() doesn't help the ESM /slim imports
+ * inside automerge-repo.
+ *
+ * This helper uses `new Function` to create a real `import()` expression
+ * that Node.js evaluates as ESM, sharing the same module graph as the
+ * Repo's internal imports.
+ */
+const dynamicImport = new Function("specifier", "return import(specifier)") as (
+  specifier: string,
+) => Promise<any>;
+
+/**
+ * Initialize the Subduction Wasm module and return the Repo constructor.
+ *
+ * The Repo constructor calls set_subduction_logger() and new MemorySigner()
+ * from @automerge/automerge-subduction/slim, which require the Wasm module
+ * to be initialized first. automerge-repo exports initSubduction() to
+ * handle this — it dynamically imports the non-/slim entry (which
+ * auto-initializes the Wasm as a side effect).
+ *
+ * Both the Repo and initSubduction must be loaded via ESM dynamic import()
+ * so they share the same module graph as the Repo's internal /slim imports.
+ */
+let cachedRepoClass: typeof Repo | undefined;
+
+async function getRepoClass(): Promise<typeof Repo> {
+  if (cachedRepoClass) return cachedRepoClass;
+
+  // Import Repo and initialize Subduction Wasm via automerge-repo's
+  // initSubduction() helper. This must happen before new Repo() because
+  // the constructor calls set_subduction_logger() and new MemorySigner()
+  // which require the Wasm module to be ready.
+  //
+  // Both imports use the ESM dynamic import wrapper so they share the
+  // same module graph as the Repo's internal /slim imports.
+  const repoMod = await dynamicImport("@automerge/automerge-repo");
+  await repoMod.initSubduction();
+  cachedRepoClass = repoMod.Repo as typeof Repo;
+  return cachedRepoClass;
+}
+
+/**
+ * Scan a directory tree for 0-byte files, which indicate incomplete writes
+ * from a previous run (process exited before storage flushed). Returns true
+ * if any are found.
+ */
+async function hasCorruptStorage(dir: string): Promise<boolean> {
+  try {
+    await fs.access(dir);
+  } catch {
+    return false;
+  }
+
+  const entries = await fs.readdir(dir, { withFileTypes: true });
+  for (const entry of entries) {
+    const fullPath = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      if (await hasCorruptStorage(fullPath)) return true;
+    } else if (entry.isFile()) {
+      const stat = await fs.stat(fullPath);
+      if (stat.size === 0) return true;
+    }
+  }
+  return false;
+}
+
+/**
+ * Create an Automerge repository with configuration-based setup.
+ *
+ * When `sub` is true, uses the Subduction sync backend built into
+ * automerge-repo. The Repo manages its own SubductionSource internally —
+ * we just pass `subductionWebsocketEndpoints` and the Repo handles
+ * connection management, sync, and retries.
+ *
+ * When `sub` is false (default), uses the traditional WebSocket network
+ * adapter for sync via the automerge sync server.
  */
 export async function createRepo(
   workingDir: string,
-  config: DirectoryConfig
+  config: DirectoryConfig,
+  sub: boolean = false
 ): Promise<Repo> {
+  const RepoClass = await getRepoClass();
+
   const syncToolDir = path.join(workingDir, ".pushwork");
-  const storage = new NodeFSStorageAdapter(path.join(syncToolDir, "automerge"));
+  const automergeDir = path.join(syncToolDir, "automerge");
+
+  // Detect and recover from corrupt local storage (0-byte files left by
+  // incomplete writes from a previous run). Wipe the cache so the Repo
+  // hydrates cleanly from the sync server.
+  if (await hasCorruptStorage(automergeDir)) {
+    console.warn("[pushwork] Corrupt local storage detected, clearing cache...");
+    await fs.rm(automergeDir, { recursive: true, force: true });
+    await fs.mkdir(automergeDir, { recursive: true });
+  }
+
+  const storage = new NodeFSStorageAdapter(automergeDir);
+
+  if (sub) {
+    const endpoints: string[] = [];
+    if (config.sync_enabled && config.sync_server) {
+      endpoints.push(config.sync_server);
+    }
+
+    return new RepoClass({
+      storage,
+      subductionWebsocketEndpoints: endpoints,
+    });
+  }
 
-  const repoConfig: any = { storage };
+  // Default: WebSocket sync adapter
+  const repoConfig: RepoConfig = { storage };
 
-  // Add network adapter only if sync is enabled and server is configured
   if (config.sync_enabled && config.sync_server) {
-    const networkAdapter = new BrowserWebSocketClientAdapter(
+    // Load the WebSocket adapter via ESM dynamic import to stay in the
+    // same module graph as the Repo.
+    const wsMod = await dynamicImport("@automerge/automerge-repo-network-websocket");
+    // The websocket adapter package (subduction.8) hasn't updated its
+    // NetworkAdapter base-class types to match the repo's new
+    // NetworkAdapterInterface (which added state() and stricter
+    // EventEmitter generics). At runtime the adapter has all required
+    // methods; this is purely a declaration mismatch.
+    const networkAdapter = new wsMod.BrowserWebSocketClientAdapter(
       config.sync_server
-    );
+    ) as unknown as NetworkAdapterInterface;
     repoConfig.network = [networkAdapter];
   }
 
-  return new Repo(repoConfig);
+  return new RepoClass(repoConfig);
 }