pushwork 1.1.8 → 1.2.3

package/src/core/sync-engine.ts

@@ -197,16 +197,25 @@ export class SyncEngine {
 
 	/**
 	 * Get the appropriate URL for a subdirectory's directory entry.
-	 * Always uses plain URLs — versioned URLs on directories can cause
-	 * issues where consumers see a version without the docs array.
+	 * Artifact directories get versioned URLs (with heads) so consumers can
+	 * fetch the exact snapshotted version, matching how artifact files work.
+	 * Non-artifact directories get plain URLs for collaborative editing.
 	 */
-	private getDirEntryUrl(handle: DocHandle<unknown>): AutomergeUrl {
+	private getDirEntryUrl(handle: DocHandle<unknown>, dirPath: string): AutomergeUrl {
+		if (this.isArtifactPath(dirPath)) {
+			return this.getVersionedUrl(handle)
+		}
 		return getPlainUrl(handle.url)
 	}
 
 	/**
 	 * Set the root directory URL in the snapshot
 	 */
+	async getRootDirectoryUrl(): Promise<AutomergeUrl | undefined> {
+		const snapshot = await this.snapshotManager.load()
+		return snapshot?.rootDirectoryUrl
+	}
+
 	async setRootDirectoryUrl(url: AutomergeUrl): Promise<void> {
 		let snapshot = await this.snapshotManager.load()
 		if (!snapshot) {
@@ -423,7 +432,7 @@ export class SyncEngine {
 	/**
 	 * Run full bidirectional sync
 	 */
-	async sync(): Promise<SyncResult> {
+	async sync(options?: {sub?: boolean}): Promise<SyncResult> {
 		const result: SyncResult = {
 			success: false,
 			filesChanged: 0,
@@ -482,7 +491,6 @@ export class SyncEngine {
 					await waitForBidirectionalSync(
 						this.repo,
 						snapshot.rootDirectoryUrl,
-						this.config.sync_server_storage_id,
 						{
 							timeoutMs: 5000, // Increased timeout for initial sync
 							pollIntervalMs: 100,
@@ -526,6 +534,12 @@ export class SyncEngine {
 
 			// Wait for network sync (important for clone scenarios)
 			if (this.config.sync_enabled) {
+				const sub = options?.sub ?? false
+				// In Subduction mode, pass no StorageId so waitForSync
+				// falls back to head-stability polling. In WebSocket mode,
+				// pass the StorageId for precise getSyncInfo-based verification.
+				const storageId = sub ? undefined : this.config.sync_server_storage_id
+
 				try {
 					// Ensure root directory handle is tracked for sync
 					if (snapshot.rootDirectoryUrl) {
@@ -546,11 +560,13 @@ export class SyncEngine {
 						out.update(`Uploading ${allHandles.length} documents to sync server`)
 						const {failed} = await waitForSync(
 							allHandles,
-							this.config.sync_server_storage_id
+							storageId
 						)
 
-						// Recreate failed documents and retry once
-						if (failed.length > 0) {
+						// Recreate failed documents and retry once.
+						// Skip in Subduction mode — SubductionSource has its
+						// own heal-sync retry logic.
+						if (failed.length > 0 && !sub) {
 							debug(`sync: ${failed.length} documents failed, recreating`)
 							out.update(`Recreating ${failed.length} failed documents`)
 							const retryHandles = await this.recreateFailedDocuments(failed, snapshot)
@@ -559,7 +575,7 @@ export class SyncEngine {
 								out.update(`Retrying ${retryHandles.length} recreated documents`)
 								const retry = await waitForSync(
 									retryHandles,
-									this.config.sync_server_storage_id
+									storageId
 								)
 								if (retry.failed.length > 0) {
 									const msg = `${retry.failed.length} documents failed to sync to server after recreation`
@@ -572,6 +588,11 @@ export class SyncEngine {
 									})
 								}
 							}
+						} else if (failed.length > 0 && sub) {
+							const msg = `${failed.length} document${failed.length === 1 ? '' : 's'} did not converge during sync (Subduction will retry in the background; re-run sync to confirm)`
+							debug(`sync: ${msg}`)
+							out.taskLine(msg, true)
+							result.warnings.push(msg)
 						}
 
 						debug("sync: all handles synced to server")
@@ -585,7 +606,6 @@ export class SyncEngine {
 					await waitForBidirectionalSync(
 						this.repo,
 						snapshot.rootDirectoryUrl,
-						this.config.sync_server_storage_id,
 						{
 							timeoutMs: BIDIRECTIONAL_SYNC_TIMEOUT_MS,
 							pollIntervalMs: 100,
@@ -608,10 +628,15 @@ export class SyncEngine {
 							)
 						debug("sync: syncing root directory touch to server")
 						out.update("Syncing root directory update")
-						await waitForSync(
+						const rootSync = await waitForSync(
 							[rootHandle],
-							this.config.sync_server_storage_id
+							storageId
 						)
+						if (rootSync.failed.length > 0) {
+							const msg = "Root directory update did not converge to server; consumers may not see recent changes until next sync"
+							debug(`sync: ${msg}`)
+							result.warnings.push(msg)
+						}
 					}
 				} catch (error) {
 					debug(`sync: network sync error: ${error}`)
@@ -938,7 +963,7 @@ export class SyncEngine {
 							)
 						subdirUpdates.push({
 							name: childName,
-							url: this.getDirEntryUrl(childHandle),
+							url: this.getDirEntryUrl(childHandle, modifiedDir),
 						})
 					}
 				}
@@ -1438,7 +1463,7 @@ export class SyncEngine {
 						this.handlesByPath.set(directoryPath, childDirHandle)
 
 						// Get appropriate URL for directory entry
-						const entryUrl = this.getDirEntryUrl(childDirHandle)
+						const entryUrl = this.getDirEntryUrl(childDirHandle, directoryPath)
 
 						// Update snapshot with discovered directory
 						this.snapshotManager.updateDirectoryEntry(snapshot, directoryPath, {
@@ -1469,7 +1494,7 @@ export class SyncEngine {
 		const dirHandle = this.repo.create(dirDoc)
 
 		// Get appropriate URL for directory entry
-		const dirEntryUrl = this.getDirEntryUrl(dirHandle)
+		const dirEntryUrl = this.getDirEntryUrl(dirHandle, directoryPath)
 
 		// Add this directory to its parent
 		// Use plain URL for mutable handle

package/src/types/config.ts

@@ -6,6 +6,7 @@ import { StorageId } from "@automerge/automerge-repo";
 export const DEFAULT_SYNC_SERVER = "wss://sync3.automerge.org";
 export const DEFAULT_SYNC_SERVER_STORAGE_ID =
   "3760df37-a4c6-4f66-9ecd-732039a9385d" as StorageId;
+export const DEFAULT_SUBDUCTION_SERVER = "wss://subduction.sync.inkandswitch.com";
 
 /**
  * Global configuration options
@@ -25,6 +26,7 @@ export interface GlobalConfig {
  */
 export interface DirectoryConfig extends GlobalConfig {
   root_directory_url?: string;
+  subduction?: boolean;
   sync_enabled: boolean;
 }
 
@@ -42,6 +44,7 @@ export interface CloneOptions extends CommandOptions {
   force?: boolean; // Overwrite existing directory
   syncServer?: string; // Custom sync server URL
   syncServerStorageId?: StorageId; // Custom sync server storage ID
+  sub?: boolean;
 }
 
 /**
@@ -83,6 +86,7 @@ export interface CheckoutOptions extends CommandOptions {
 export interface InitOptions extends CommandOptions {
   syncServer?: string;
   syncServerStorageId?: StorageId;
+  sub?: boolean;
 }
 
 /**

package/src/utils/content.ts

@@ -16,7 +16,7 @@ export function isContentEqual(
   content2: string | Uint8Array | null
 ): boolean {
   if (content1 === content2) return true;
-  if (!content1 || !content2) return false;
+  if (content1 == null || content2 == null) return false;
 
   if (typeof content1 !== typeof content2) return false;
 

package/src/utils/network-sync.ts

@@ -21,13 +21,11 @@ function debug(...args: any[]) {
  *
  * @param repo - The Automerge repository
  * @param rootDirectoryUrl - The root directory URL to start traversal from
- * @param syncServerStorageId - The sync server storage ID
  * @param options - Configuration options
  */
 export async function waitForBidirectionalSync(
   repo: Repo,
   rootDirectoryUrl: AutomergeUrl | undefined,
-  syncServerStorageId: StorageId | undefined,
   options: {
     timeoutMs?: number;
     pollIntervalMs?: number;
@@ -42,7 +40,7 @@ export async function waitForBidirectionalSync(
     handles,
   } = options;
 
-  if (!syncServerStorageId || !rootDirectoryUrl) {
+  if (!rootDirectoryUrl) {
     return;
   }
 
@@ -295,16 +293,20 @@ export async function waitForSync(
 ): Promise<SyncWaitResult> {
   const startTime = Date.now();
 
-  if (!syncServerStorageId) {
-    debug("waitForSync: no sync server storage ID, skipping");
-    return { failed: [] };
-  }
-
   if (handlesToWaitOn.length === 0) {
     debug("waitForSync: no documents to sync");
     return { failed: [] };
   }
 
+  // When no StorageId is available (Subduction mode), use head-stability
+  // polling. The SubductionSource handles sync internally — we just wait
+  // for each handle's heads to stop changing.
+  if (!syncServerStorageId) {
+    debug(`waitForSync: no storage ID, using head-stability polling for ${handlesToWaitOn.length} documents`);
+    out.taskLine(`Waiting for ${handlesToWaitOn.length} documents to sync`);
+    return waitForSyncViaHeadStability(handlesToWaitOn, timeoutMs, startTime);
+  }
+
   debug(`waitForSync: waiting for ${handlesToWaitOn.length} documents (timeout=${timeoutMs}ms, batchSize=${SYNC_BATCH_SIZE})`);
 
   // Separate already-synced from needs-sync
@@ -370,3 +372,85 @@ export async function waitForSync(
 
   return { failed };
 }
+
+/**
+ * Wait for sync by polling head stability (Subduction mode).
+ * Each handle's heads are polled until they remain unchanged for
+ * several consecutive checks, indicating the SubductionSource has
+ * finished syncing.
+ */
+async function waitForSyncViaHeadStability(
+  handles: DocHandle<unknown>[],
+  timeoutMs: number,
+  startTime: number,
+): Promise<SyncWaitResult> {
+  const failed: DocHandle<unknown>[] = [];
+  let synced = 0;
+
+  // Process in batches
+  for (let i = 0; i < handles.length; i += SYNC_BATCH_SIZE) {
+    const batch = handles.slice(i, i + SYNC_BATCH_SIZE);
+
+    const results = await Promise.allSettled(
+      batch.map(handle => waitForHandleHeadStability(handle, timeoutMs, startTime))
+    );
+
+    for (const result of results) {
+      if (result.status === "rejected") {
+        failed.push(result.reason as DocHandle<unknown>);
+      } else {
+        synced++;
+      }
+    }
+  }
+
+  const elapsed = Date.now() - startTime;
+  if (failed.length > 0) {
+    debug(`waitForSync(heads): ${failed.length} documents failed after ${elapsed}ms`);
+    out.taskLine(`Sync: ${synced} synced, ${failed.length} timed out after ${(elapsed / 1000).toFixed(1)}s`, true);
+  } else {
+    debug(`waitForSync(heads): all ${handles.length} documents synced in ${elapsed}ms`);
+    out.taskLine(`All ${handles.length} documents synced (${(elapsed / 1000).toFixed(1)}s)`);
+  }
+
+  return { failed };
+}
+
+/**
+ * Wait for a single handle's heads to stabilize.
+ * Polls heads at 100ms intervals; resolves after 3 consecutive stable
+ * checks, rejects on timeout.
+ */
+function waitForHandleHeadStability(
+  handle: DocHandle<unknown>,
+  timeoutMs: number,
+  startTime: number,
+): Promise<DocHandle<unknown>> {
+  return new Promise<DocHandle<unknown>>((resolve, reject) => {
+    let lastHeads = JSON.stringify(handle.heads());
+    let stableCount = 0;
+    const stableRequired = 3;
+
+    const pollInterval = setInterval(() => {
+      const currentHeads = JSON.stringify(handle.heads());
+      if (currentHeads === lastHeads) {
+        stableCount++;
+        if (stableCount >= stableRequired) {
+          clearInterval(pollInterval);
+          clearTimeout(timeout);
+          debug(`waitForSync(heads): ${handle.url}... converged in ${Date.now() - startTime}ms`);
+          resolve(handle);
+        }
+      } else {
+        stableCount = 0;
+        lastHeads = currentHeads;
+      }
+    }, 100);
+
+    const timeout = setTimeout(() => {
+      clearInterval(pollInterval);
+      debug(`waitForSync(heads): ${handle.url}... timed out after ${timeoutMs}ms`);
+      reject(handle);
+    }, timeoutMs);
+  });
+}

package/src/utils/output.ts

@@ -345,7 +345,7 @@ export class Output {
       this.taskOriginalMessage = null;
       this.taskCurrentMessage = null;
     }
-    console.log(
+    console.error(
       chalk.red(
         message instanceof Error
           ? message.message
@@ -367,7 +367,7 @@ export class Output {
       this.taskOriginalMessage = null;
       this.taskCurrentMessage = null;
     }
-    console.log(
+    console.error(
       `\n${chalk.bgRed.white(` ${label} `)}${message && ` ${message}`}`
     );
   }
@@ -400,19 +400,19 @@ export class Output {
 
     if (error instanceof Error) {
       // Error type and message
-      console.log(chalk.red(`${error.name}: ${error.message}`));
+      console.error(chalk.red(`${error.name}: ${error.message}`));
 
       // Stack trace
       if (error.stack) {
-        console.log("");
-        console.log(chalk.dim("Stack trace:"));
+        console.error("");
+        console.error(chalk.dim("Stack trace:"));
         const stackLines = error.stack.split("\n").slice(1); // Skip first line (error message)
         stackLines.forEach((line) =>
-          console.log(chalk.dim(`  ${line.trim()}`))
+          console.error(chalk.dim(`  ${line.trim()}`))
         );
       }
     } else {
-      console.log(chalk.red(String(error)));
+      console.error(chalk.red(String(error)));
     }
 
     process.exit(exitCode);

package/src/utils/repo-factory.ts

@@ -1,28 +1,149 @@
-import { Repo } from "@automerge/automerge-repo";
-import { NodeFSStorageAdapter } from "@automerge/automerge-repo-storage-nodefs";
-import { BrowserWebSocketClientAdapter } from "@automerge/automerge-repo-network-websocket";
-import * as path from "path";
-import { DirectoryConfig } from "../types";
+import {
+	type Repo,
+	type RepoConfig,
+	type NetworkAdapterInterface,
+} from "@automerge/automerge-repo"
+import {NodeFSStorageAdapter} from "@automerge/automerge-repo-storage-nodefs"
+import * as fs from "fs/promises"
+import * as path from "path"
+import {DirectoryConfig} from "../types"
 
 /**
- * Create an Automerge repository with configuration-based setup
+ * Perform a real ESM dynamic import that tsc won't rewrite to require().
+ *
+ * TypeScript with `"module": "commonjs"` compiles `await import("x")` to
+ * `require("x")`, which resolves CJS entries instead of ESM entries. The
+ * Wasm module instance is different between the CJS and ESM module graphs,
+ * so initializing via CJS require() doesn't help the ESM /slim imports
+ * inside automerge-repo.
+ *
+ * This helper uses `new Function` to create a real `import()` expression
+ * that Node.js evaluates as ESM, sharing the same module graph as the
+ * Repo's internal imports.
+ */
+const dynamicImport = new Function("specifier", "return import(specifier)") as (
+	specifier: string
+) => Promise<any>
+
+/**
+ * Initialize the Subduction Wasm module and return the Repo constructor.
+ *
+ * The Repo constructor calls set_subduction_logger() and new MemorySigner()
+ * from @automerge/automerge-subduction/slim, which require the Wasm module
+ * to be initialized first. automerge-repo exports initSubduction() to
+ * handle this — it dynamically imports the non-/slim entry (which
+ * auto-initializes the Wasm as a side effect).
+ *
+ * Both the Repo and initSubduction must be loaded via ESM dynamic import()
+ * so they share the same module graph as the Repo's internal /slim imports.
+ */
+let cachedRepoClass: typeof Repo | undefined
+
+async function getRepoClass(): Promise<typeof Repo> {
+	if (cachedRepoClass) return cachedRepoClass
+
+	// Import Repo and initialize Subduction Wasm via automerge-repo's
+	// initSubduction() helper. This must happen before new Repo() because
+	// the constructor calls set_subduction_logger() and new MemorySigner()
+	// which require the Wasm module to be ready.
+	//
+	// Both imports use the ESM dynamic import wrapper so they share the
+	// same module graph as the Repo's internal /slim imports.
+	const repoMod = await dynamicImport("@automerge/automerge-repo")
+	await repoMod.initSubduction()
+	cachedRepoClass = repoMod.Repo as typeof Repo
+	return cachedRepoClass
+}
+
+/**
+ * Scan a directory tree for 0-byte files, which indicate incomplete writes
+ * from a previous run (process exited before storage flushed). Returns true
+ * if any are found.
+ */
+async function hasCorruptStorage(dir: string): Promise<boolean> {
+	try {
+		await fs.access(dir)
+	} catch {
+		return false
+	}
+
+	const entries = await fs.readdir(dir, {withFileTypes: true})
+	for (const entry of entries) {
+		const fullPath = path.join(dir, entry.name)
+		if (entry.isDirectory()) {
+			if (await hasCorruptStorage(fullPath)) return true
+		} else if (entry.isFile()) {
+			const stat = await fs.stat(fullPath)
+			if (stat.size === 0) return true
+		}
+	}
+	return false
+}
+
+/**
+ * Create an Automerge repository with configuration-based setup.
+ *
+ * When `sub` is true, uses the Subduction sync backend built into
+ * automerge-repo. The Repo manages its own SubductionSource internally —
+ * we just pass `subductionWebsocketEndpoints` and the Repo handles
+ * connection management, sync, and retries.
+ *
+ * When `sub` is false (default), uses the traditional WebSocket network
+ * adapter for sync via the automerge sync server.
  */
 export async function createRepo(
-  workingDir: string,
-  config: DirectoryConfig
+	workingDir: string,
+	config: DirectoryConfig,
+	sub: boolean = false
 ): Promise<Repo> {
-  const syncToolDir = path.join(workingDir, ".pushwork");
-  const storage = new NodeFSStorageAdapter(path.join(syncToolDir, "automerge"));
+	const RepoClass = await getRepoClass()
+
+	const syncToolDir = path.join(workingDir, ".pushwork")
+	const automergeDir = path.join(syncToolDir, "automerge")
+
+	// Detect and recover from corrupt local storage (0-byte files left by
+	// incomplete writes from a previous run). Wipe the cache so the Repo
+	// hydrates cleanly from the sync server.
+	if (await hasCorruptStorage(automergeDir)) {
+		console.warn("[pushwork] Corrupt local storage detected, clearing cache...")
+		await fs.rm(automergeDir, {recursive: true, force: true})
+		await fs.mkdir(automergeDir, {recursive: true})
+	}
+
+	const storage = new NodeFSStorageAdapter(automergeDir)
+
+	if (sub) {
+		const endpoints: string[] = []
+		if (config.sync_enabled && config.sync_server) {
+			endpoints.push(config.sync_server)
+		}
+
+		return new RepoClass({
+			storage,
+			// @ts-expect-error i don't know why
+			subductionWebsocketEndpoints: endpoints,
+		})
+	}
 
-  const repoConfig: any = { storage };
+	// Default: WebSocket sync adapter
+	const repoConfig: RepoConfig = {storage}
 
-  // Add network adapter only if sync is enabled and server is configured
-  if (config.sync_enabled && config.sync_server) {
-    const networkAdapter = new BrowserWebSocketClientAdapter(
-      config.sync_server
-    );
-    repoConfig.network = [networkAdapter];
-  }
+	if (config.sync_enabled && config.sync_server) {
+		// Load the WebSocket adapter via ESM dynamic import to stay in the
+		// same module graph as the Repo.
+		const wsMod = await dynamicImport(
+			"@automerge/automerge-repo-network-websocket"
+		)
+		// The websocket adapter package (subduction.8) hasn't updated its
+		// NetworkAdapter base-class types to match the repo's new
+		// NetworkAdapterInterface (which added state() and stricter
+		// EventEmitter generics). At runtime the adapter has all required
+		// methods; this is purely a declaration mismatch.
+		const networkAdapter = new wsMod.BrowserWebSocketClientAdapter(
+			config.sync_server
+		) as unknown as NetworkAdapterInterface
+		repoConfig.network = [networkAdapter]
+	}
 
-  return new Repo(repoConfig);
+	return new RepoClass(repoConfig)
 }