wispy-cli 1.2.3 → 1.4.0

package/core/sync.mjs

@@ -0,0 +1,682 @@
+/**
+ * core/sync.mjs — Sync manager for Wispy
+ *
+ * Syncs sessions, memory, cron jobs, workstreams, and permissions
+ * between local and remote wispy instances via HTTP API.
+ *
+ * Protocol:
+ *   1. Build local manifest (path → { hash, modifiedAt, size })
+ *   2. Fetch remote manifest from GET /api/sync/manifest
+ *   3. Compare → compute what to push / pull
+ *   4. Transfer only changed files
+ *   5. Report summary
+ */
+
+import path from "node:path";
+import { readFile, writeFile, mkdir, readdir, stat } from "node:fs/promises";
+import { createHash } from "node:crypto";
+
+import { WISPY_DIR } from "./config.mjs";
+
+const SYNC_CONFIG_FILE = path.join(WISPY_DIR, "sync.json");
+
+// File patterns that are syncable (relative to WISPY_DIR)
+const SYNC_PATTERNS = [
+  { glob: "memory",         match: (f) => f.startsWith("memory/") && f.endsWith(".md") },
+  { glob: "sessions",       match: (f) => f.startsWith("sessions/") && f.endsWith(".json") },
+  { glob: "cron/jobs.json", match: (f) => f === "cron/jobs.json" },
+  { glob: "workstreams",    match: (f) => f.match(/^workstreams\/[^/]+\/work\.md$/) },
+  { glob: "permissions.json", match: (f) => f === "permissions.json" },
+];
+
+/**
+ * Compute SHA-256 hash of a string or Buffer
+ */
+function sha256(content) {
+  return createHash("sha256").update(content).digest("hex");
+}
+
+/**
+ * @typedef {Object} FileEntry
+ * @property {string} path       - relative path within WISPY_DIR
+ * @property {string} hash       - SHA-256 of content
+ * @property {string} modifiedAt - ISO timestamp
+ * @property {number} size       - byte size
+ */
+
+/**
+ * @typedef {Object} SyncResult
+ * @property {number} pushed
+ * @property {number} pulled
+ * @property {number} skipped
+ * @property {number} conflicts
+ * @property {string[]} errors
+ * @property {Object} details
+ */
+
+export class SyncManager {
+  /**
+   * @param {object} config
+   * @param {string} [config.remoteUrl]
+   * @param {string} [config.token]
+   * @param {'newer-wins'|'local-wins'|'remote-wins'} [config.strategy]
+   * @param {boolean} [config.auto]
+   */
+  constructor(config = {}) {
+    this.remoteUrl = config.remoteUrl ?? null;
+    this.token = config.token ?? null;
+    this.strategy = config.strategy ?? "newer-wins";
+    this.auto = config.auto ?? false;
+  }
+
+  // ── Config persistence ──────────────────────────────────────────────────────
+
+  static async loadConfig() {
+    try {
+      const raw = await readFile(SYNC_CONFIG_FILE, "utf8");
+      return JSON.parse(raw);
+    } catch {
+      return {};
+    }
+  }
+
+  static async saveConfig(cfg) {
+    await mkdir(WISPY_DIR, { recursive: true });
+    await writeFile(SYNC_CONFIG_FILE, JSON.stringify(cfg, null, 2) + "\n", "utf8");
+  }
+
+  static async fromConfig() {
+    const cfg = await SyncManager.loadConfig();
+    return new SyncManager(cfg);
+  }
+
+  // ── Local manifest ──────────────────────────────────────────────────────────
+
+  /**
+   * Scan WISPY_DIR and build manifest for syncable files.
+   * @param {object} [opts]
+   * @param {boolean} [opts.memoryOnly]
+   * @param {boolean} [opts.sessionsOnly]
+   * @returns {Promise<FileEntry[]>}
+   */
+  async buildLocalManifest(opts = {}) {
+    const entries = [];
+    await this._scanDir(WISPY_DIR, WISPY_DIR, entries, opts);
+    return entries;
+  }
+
+  async _scanDir(baseDir, dir, entries, opts = {}) {
+    let items;
+    try {
+      items = await readdir(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+
+    for (const item of items) {
+      const fullPath = path.join(dir, item.name);
+      const relPath = path.relative(baseDir, fullPath).replace(/\\/g, "/");
+
+      if (item.isDirectory()) {
+        // Skip node_modules, .git, etc.
+        if (["node_modules", ".git", ".npm"].includes(item.name)) continue;
+        await this._scanDir(baseDir, fullPath, entries, opts);
+      } else if (item.isFile()) {
+        // Check if syncable
+        const syncable = SYNC_PATTERNS.some(p => p.match(relPath));
+        if (!syncable) continue;
+
+        // Apply filters
+        if (opts.memoryOnly && !relPath.startsWith("memory/")) continue;
+        if (opts.sessionsOnly && !relPath.startsWith("sessions/")) continue;
+
+        try {
+          const content = await readFile(fullPath);
+          const fileStat = await stat(fullPath);
+          entries.push({
+            path: relPath,
+            hash: sha256(content),
+            modifiedAt: fileStat.mtime.toISOString(),
+            size: content.length,
+          });
+        } catch {
+          // Skip unreadable files
+        }
+      }
+    }
+  }
+
+  // ── HTTP helpers ────────────────────────────────────────────────────────────
+
+  async _fetch(url, opts = {}) {
+    const { default: fetch } = await import("node:fetch").catch(async () => {
+      // Node 18+ has built-in fetch
+      return { default: globalThis.fetch };
+    });
+
+    const headers = {
+      "Content-Type": "application/json",
+      ...(this.token ? { Authorization: `Bearer ${this.token}` } : {}),
+      ...opts.headers,
+    };
+
+    const response = await fetch(url, { ...opts, headers });
+    return response;
+  }
+
+  async _getRemoteManifest(remoteUrl, token) {
+    const url = `${remoteUrl}/api/sync/manifest`;
+    const res = await this._fetchWithAuth(url, {}, token);
+    if (!res.ok) {
+      throw new Error(`Failed to get remote manifest: ${res.status} ${res.statusText}`);
+    }
+    const data = await res.json();
+    return data.files ?? [];
+  }
+
+  async _downloadFile(remoteUrl, token, relPath) {
+    const url = `${remoteUrl}/api/sync/file?path=${encodeURIComponent(relPath)}`;
+    const res = await this._fetchWithAuth(url, {}, token);
+    if (!res.ok) {
+      throw new Error(`Failed to download ${relPath}: ${res.status}`);
+    }
+    return res.json(); // { path, content, modifiedAt, hash, encoding }
+  }
+
+  async _uploadFile(remoteUrl, token, entry) {
+    const url = `${remoteUrl}/api/sync/file`;
+    const res = await this._fetchWithAuth(url, {
+      method: "POST",
+      body: JSON.stringify(entry),
+    }, token);
+    if (!res.ok) {
+      const text = await res.text().catch(() => "");
+      throw new Error(`Failed to upload ${entry.path}: ${res.status} ${text}`);
+    }
+    return res.json();
+  }
+
+  async _uploadBulk(remoteUrl, token, files) {
+    const url = `${remoteUrl}/api/sync/bulk`;
+    const res = await this._fetchWithAuth(url, {
+      method: "POST",
+      body: JSON.stringify({ files }),
+    }, token);
+    if (!res.ok) {
+      const text = await res.text().catch(() => "");
+      throw new Error(`Bulk upload failed: ${res.status} ${text}`);
+    }
+    return res.json();
+  }
+
+  async _fetchWithAuth(url, opts = {}, tokenOverride = null) {
+    const token = tokenOverride ?? this.token;
+    const headers = {
+      "Content-Type": "application/json",
+      ...(token ? { Authorization: `Bearer ${token}` } : {}),
+      ...(opts.headers ?? {}),
+    };
+
+    // Use native fetch (Node 18+) or fallback
+    const fetchFn = globalThis.fetch;
+    if (!fetchFn) {
+      throw new Error("fetch not available. Node 18+ required.");
+    }
+
+    return fetchFn(url, { ...opts, headers });
+  }
+
+  // ── Core operations ─────────────────────────────────────────────────────────
+
+  /**
+   * Push local state to remote.
+   * @param {string} remoteUrl
+   * @param {string} token
+   * @param {object} [opts]
+   * @returns {Promise<SyncResult>}
+   */
+  async push(remoteUrl, token, opts = {}) {
+    const local = await this.buildLocalManifest(opts);
+    const remote = await this._getRemoteManifest(remoteUrl, token);
+    const remoteMap = new Map(remote.map(f => [f.path, f]));
+
+    const toUpload = [];
+    let skipped = 0;
+
+    for (const localFile of local) {
+      const remoteFile = remoteMap.get(localFile.path);
+      if (!remoteFile) {
+        toUpload.push(localFile);
+        continue;
+      }
+      if (localFile.hash === remoteFile.hash) {
+        skipped++;
+        continue;
+      }
+      // Both exist, different content
+      const strategy = opts.strategy ?? this.strategy;
+      if (strategy === "local-wins") {
+        toUpload.push(localFile);
+      } else if (strategy === "remote-wins") {
+        skipped++;
+      } else {
+        // newer-wins: push only if local is newer
+        const localMtime = new Date(localFile.modifiedAt).getTime();
+        const remoteMtime = new Date(remoteFile.modifiedAt).getTime();
+        if (localMtime > remoteMtime) {
+          toUpload.push(localFile);
+        } else {
+          skipped++;
+        }
+      }
+    }
+
+    // Upload files
+    const errors = [];
+    let pushed = 0;
+
+    if (toUpload.length > 0) {
+      // Prepare payload with content
+      const files = [];
+      for (const f of toUpload) {
+        try {
+          const content = await readFile(path.join(WISPY_DIR, f.path));
+          files.push({
+            path: f.path,
+            content: content.toString("base64"),
+            encoding: "base64",
+            modifiedAt: f.modifiedAt,
+            hash: f.hash,
+          });
+        } catch (err) {
+          errors.push(`Read failed for ${f.path}: ${err.message}`);
+        }
+      }
+
+      if (files.length > 0) {
+        try {
+          await this._uploadBulk(remoteUrl, token, files);
+          pushed = files.length;
+        } catch (err) {
+          // Fallback: upload one by one
+          for (const file of files) {
+            try {
+              await this._uploadFile(remoteUrl, token, file);
+              pushed++;
+            } catch (e) {
+              errors.push(`Upload failed for ${file.path}: ${e.message}`);
+            }
+          }
+        }
+      }
+    }
+
+    return {
+      pushed,
+      pulled: 0,
+      skipped,
+      conflicts: 0,
+      errors,
+      details: { uploaded: toUpload.map(f => f.path) },
+    };
+  }
+
+  /**
+   * Pull remote state to local.
+   * @param {string} remoteUrl
+   * @param {string} token
+   * @param {object} [opts]
+   * @returns {Promise<SyncResult>}
+   */
+  async pull(remoteUrl, token, opts = {}) {
+    const local = await this.buildLocalManifest(opts);
+    const remote = await this._getRemoteManifest(remoteUrl, token);
+    const localMap = new Map(local.map(f => [f.path, f]));
+
+    const toDownload = [];
+    let skipped = 0;
+    let conflicts = 0;
+
+    for (const remoteFile of remote) {
+      const localFile = localMap.get(remoteFile.path);
+      if (!localFile) {
+        toDownload.push(remoteFile);
+        continue;
+      }
+      if (localFile.hash === remoteFile.hash) {
+        skipped++;
+        continue;
+      }
+      // Both exist, different content
+      const strategy = opts.strategy ?? this.strategy;
+      if (strategy === "remote-wins") {
+        toDownload.push(remoteFile);
+      } else if (strategy === "local-wins") {
+        skipped++;
+      } else {
+        // newer-wins
+        const localMtime = new Date(localFile.modifiedAt).getTime();
+        const remoteMtime = new Date(remoteFile.modifiedAt).getTime();
+        if (remoteMtime > localMtime) {
+          toDownload.push(remoteFile);
+        } else if (localMtime === remoteMtime) {
+          // Same timestamp, different content → conflict
+          conflicts++;
+          const conflictPath = path.join(WISPY_DIR, remoteFile.path + `.conflict-${Date.now()}`);
+          try {
+            const downloaded = await this._downloadFile(remoteUrl, token, remoteFile.path);
+            const content = Buffer.from(downloaded.content, downloaded.encoding === "base64" ? "base64" : "utf8");
+            await mkdir(path.dirname(conflictPath), { recursive: true });
+            await writeFile(conflictPath, content);
+          } catch {}
+        } else {
+          skipped++;
+        }
+      }
+    }
+
+    // Download files
+    const errors = [];
+    let pulled = 0;
+
+    for (const remoteFile of toDownload) {
+      try {
+        const downloaded = await this._downloadFile(remoteUrl, token, remoteFile.path);
+        const content = Buffer.from(
+          downloaded.content,
+          downloaded.encoding === "base64" ? "base64" : "utf8"
+        );
+        const localPath = path.join(WISPY_DIR, remoteFile.path);
+        await mkdir(path.dirname(localPath), { recursive: true });
+        await writeFile(localPath, content);
+        // Restore mtime
+        const mtime = new Date(remoteFile.modifiedAt);
+        const { utimes } = await import("node:fs/promises");
+        await utimes(localPath, mtime, mtime).catch(() => {});
+        pulled++;
+      } catch (err) {
+        errors.push(`Download failed for ${remoteFile.path}: ${err.message}`);
+      }
+    }
+
+    return {
+      pushed: 0,
+      pulled,
+      skipped,
+      conflicts,
+      errors,
+      details: { downloaded: toDownload.map(f => f.path) },
+    };
+  }
+
+  /**
+   * Bidirectional sync — push local-only/newer files, pull remote-only/newer files.
+   * @param {string} remoteUrl
+   * @param {string} token
+   * @param {object} [opts]
+   * @returns {Promise<SyncResult>}
+   */
+  async sync(remoteUrl, token, opts = {}) {
+    const local = await this.buildLocalManifest(opts);
+    const remote = await this._getRemoteManifest(remoteUrl, token);
+
+    const localMap = new Map(local.map(f => [f.path, f]));
+    const remoteMap = new Map(remote.map(f => [f.path, f]));
+
+    const toUpload = [];
+    const toDownload = [];
+    let skipped = 0;
+    let conflicts = 0;
+    const errors = [];
+    const strategy = opts.strategy ?? this.strategy;
+
+    // Check local files
+    for (const localFile of local) {
+      const remoteFile = remoteMap.get(localFile.path);
+      if (!remoteFile) {
+        toUpload.push(localFile);
+        continue;
+      }
+      if (localFile.hash === remoteFile.hash) {
+        skipped++;
+        continue;
+      }
+      // Both exist, different content
+      const localMtime = new Date(localFile.modifiedAt).getTime();
+      const remoteMtime = new Date(remoteFile.modifiedAt).getTime();
+
+      if (strategy === "local-wins") {
+        toUpload.push(localFile);
+      } else if (strategy === "remote-wins") {
+        toDownload.push(remoteFile);
+      } else if (localMtime === remoteMtime) {
+        // Conflict
+        conflicts++;
+        // Keep both: rename remote as conflict copy
+        const conflictPath = remoteFile.path + `.conflict-${Date.now()}`;
+        toDownload.push({ ...remoteFile, path: conflictPath });
+      } else if (localMtime > remoteMtime) {
+        toUpload.push(localFile);
+      } else {
+        toDownload.push(remoteFile);
+      }
+    }
+
+    // Check remote-only files
+    for (const remoteFile of remote) {
+      if (!localMap.has(remoteFile.path)) {
+        toDownload.push(remoteFile);
+      }
+    }
+
+    // Upload
+    let pushed = 0;
+    if (toUpload.length > 0) {
+      const files = [];
+      for (const f of toUpload) {
+        try {
+          const content = await readFile(path.join(WISPY_DIR, f.path));
+          files.push({
+            path: f.path,
+            content: content.toString("base64"),
+            encoding: "base64",
+            modifiedAt: f.modifiedAt,
+            hash: f.hash,
+          });
+        } catch (err) {
+          errors.push(`Read failed for ${f.path}: ${err.message}`);
+        }
+      }
+      if (files.length > 0) {
+        try {
+          await this._uploadBulk(remoteUrl, token, files);
+          pushed = files.length;
+        } catch (err) {
+          for (const file of files) {
+            try {
+              await this._uploadFile(remoteUrl, token, file);
+              pushed++;
+            } catch (e) {
+              errors.push(`Upload failed for ${file.path}: ${e.message}`);
+            }
+          }
+        }
+      }
+    }
+
+    // Download
+    let pulled = 0;
+    for (const remoteFile of toDownload) {
+      try {
+        const downloaded = await this._downloadFile(remoteUrl, token, remoteFile.path);
+        const content = Buffer.from(
+          downloaded.content,
+          downloaded.encoding === "base64" ? "base64" : "utf8"
+        );
+        const localPath = path.join(WISPY_DIR, remoteFile.path);
+        await mkdir(path.dirname(localPath), { recursive: true });
+        await writeFile(localPath, content);
+        const mtime = new Date(remoteFile.modifiedAt);
+        const { utimes } = await import("node:fs/promises");
+        await utimes(localPath, mtime, mtime).catch(() => {});
+        pulled++;
+      } catch (err) {
+        errors.push(`Download failed for ${remoteFile.path}: ${err.message}`);
+      }
+    }
+
+    return {
+      pushed,
+      pulled,
+      skipped,
+      conflicts,
+      errors,
+      details: {
+        uploaded: toUpload.map(f => f.path),
+        downloaded: toDownload.map(f => f.path),
+      },
+    };
+  }
+
+  /**
+   * Dry-run: show what would sync without doing it.
+   * @param {string} remoteUrl
+   * @param {string} token
+   * @param {object} [opts]
+   * @returns {Promise<object>} status info
+   */
+  async status(remoteUrl, token, opts = {}) {
+    let remote;
+    let reachable = true;
+    try {
+      remote = await this._getRemoteManifest(remoteUrl, token);
+    } catch (err) {
+      reachable = false;
+      remote = [];
+    }
+
+    const local = await this.buildLocalManifest(opts);
+    const localMap = new Map(local.map(f => [f.path, f]));
+    const remoteMap = new Map(remote.map(f => [f.path, f]));
+
+    const byType = (prefix, checkFn) => {
+      const files = local.filter(f => checkFn ? checkFn(f.path) : f.path.startsWith(prefix));
+      const remoteFiles = remote.filter(f => checkFn ? checkFn(f.path) : f.path.startsWith(prefix));
+
+      const localOnly = files.filter(f => !remoteMap.has(f.path));
+      const remoteOnly = remoteFiles.filter(f => !localMap.has(f.path));
+      const inSync = files.filter(f => {
+        const r = remoteMap.get(f.path);
+        return r && r.hash === f.hash;
+      });
+
+      return { localOnly: localOnly.length, remoteOnly: remoteOnly.length, inSync: inSync.length };
+    };
+
+    return {
+      reachable,
+      remoteUrl,
+      memory: byType("memory/"),
+      sessions: byType("sessions/"),
+      cron: byType(null, f => f === "cron/jobs.json"),
+      workstreams: byType(null, f => /^workstreams\//.test(f)),
+      permissions: byType(null, f => f === "permissions.json"),
+    };
+  }
+
+  // ── Selective sync helpers ──────────────────────────────────────────────────
+
+  async pushMemory(remoteUrl, token) {
+    return this.push(remoteUrl, token, { memoryOnly: true });
+  }
+
+  async pullMemory(remoteUrl, token) {
+    return this.pull(remoteUrl, token, { memoryOnly: true });
+  }
+
+  async pushSessions(remoteUrl, token) {
+    return this.push(remoteUrl, token, { sessionsOnly: true });
+  }
+
+  async pullSessions(remoteUrl, token) {
+    return this.pull(remoteUrl, token, { sessionsOnly: true });
+  }
+
+  async pushCron(remoteUrl, token) {
+    return this.push(remoteUrl, token, { cronOnly: true });
+  }
+
+  async pullCron(remoteUrl, token) {
+    return this.pull(remoteUrl, token, { cronOnly: true });
+  }
+
+  // ── Single-file push (for auto-sync) ───────────────────────────────────────
+
+  /**
+   * Push a single file to remote (used for auto-sync hooks).
+   * Non-blocking: fires and forgets.
+   */
+  pushFile(absolutePath) {
+    if (!this.remoteUrl || !this.token) return;
+    const relPath = path.relative(WISPY_DIR, absolutePath).replace(/\\/g, "/");
+
+    // Check it's a syncable path
+    const syncable = SYNC_PATTERNS.some(p => p.match(relPath));
+    if (!syncable) return;
+
+    // Fire and forget
+    (async () => {
+      try {
+        const content = await readFile(absolutePath);
+        const fileStat = await stat(absolutePath);
+        await this._uploadFile(this.remoteUrl, this.token, {
+          path: relPath,
+          content: content.toString("base64"),
+          encoding: "base64",
+          modifiedAt: fileStat.mtime.toISOString(),
+          hash: sha256(content),
+        });
+      } catch {
+        // Silent fail for background auto-sync
+      }
+    })();
+  }
+
+  /**
+   * Enable auto-sync: save config and set auto=true.
+   */
+  static async enableAuto(remoteUrl, token) {
+    const cfg = await SyncManager.loadConfig();
+    cfg.auto = true;
+    cfg.remoteUrl = remoteUrl;
+    cfg.token = token;
+    await SyncManager.saveConfig(cfg);
+  }
+
+  /**
+   * Disable auto-sync.
+   */
+  static async disableAuto() {
+    const cfg = await SyncManager.loadConfig();
+    cfg.auto = false;
+    await SyncManager.saveConfig(cfg);
+  }
+}
+
+/**
+ * Get or create a singleton SyncManager from ~/.wispy/sync.json
+ * Returns null if not configured.
+ */
+let _instance = null;
+export async function getSyncManager() {
+  if (_instance) return _instance;
+  const cfg = await SyncManager.loadConfig();
+  if (!cfg.remoteUrl) return null;
+  _instance = new SyncManager(cfg);
+  return _instance;
+}
+
+/**
+ * Hash a file content Buffer
+ */
+export { sha256 };