syndai 0.5.0 → 0.7.0

package/src/sync.js

@@ -0,0 +1,389 @@
+// `syndai sync <dir> [--app A] [--watch]`: keep a local folder (an Obsidian vault, any folder of
+// notes) in a project's Knowledge (SPEC-knowledge-sources K6; docs: syndai.ai/docs/cli, "Sync a
+// folder into Knowledge").
+//
+// One sync: walk the folder, hash every file, send the manifest. The server deletes the files no
+// longer here and answers the paths that are new or changed; those are pushed in batches of at
+// most 500 files and 8 MB. A text file goes as its UTF-8 text, any other file by name only. The
+// folder's name in Knowledge is the directory's basename.
+//
+// Skipped: hidden files and folders (a name starting with `.`, so `.obsidian/`, `.trash/` and
+// `.git/`), `node_modules/`, symlinks (never followed: one may point out of the folder, or
+// loop), files over 10 MB, and the patterns in `<dir>/.syndaiignore`.
+//
+// `--watch` re-runs the whole sync once the folder has been quiet for a second, one sync at a
+// time; Ctrl-C stops it. A file is hashed again only when its size or mtime moved (or it was
+// hashed within 2 s of its mtime, which a coarse clock could hide), so a re-sync costs the
+// changed files, not the vault. File events (`fs.watch`) are the trigger; a stat-only
+// check every 10 s catches what they miss (a network drive, a dropped event), so an edit is in
+// Knowledge within seconds either way.
+//
+// File I/O is synchronous: one file at a time through promises is ~20x slower on a big vault,
+// and nothing else runs meanwhile.
+
+import { createHash } from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { CliError, EXIT, UnavailableError, explain } from "./errors.js";
+import { interrupted } from "./http.js";
+import { projectLabel, resolveProject } from "./projects.js";
+
+/** Sent as text; the server's `TEXT_SUFFIXES` (backend/src/features/project_sources/ingest.py). */
+export const TEXT_SUFFIXES = new Set([
+  ".md",
+  ".markdown",
+  ".mdx",
+  ".txt",
+  ".text",
+  ".csv",
+  ".tsv",
+  ".json",
+  ".yaml",
+  ".yml",
+  ".org",
+  ".rst",
+  ".log",
+]);
+export const FILE_MAX_BYTES = 10 * 1024 * 1024;
+export const MANIFEST_MAX = 20_000;
+export const BATCH_FILES = 500;
+export const BATCH_BYTES = 8 * 1024 * 1024;
+export const DEBOUNCE_MS = 1000;
+const POLL_MS = 10_000;
+const RETRY_MS = 30_000;
+const RACY_MS = 2000; // the coarsest common mtime clock (FAT, exFAT)
+const TOO_BIG = "over 10 MB";
+const IGNORE_FILE = ".syndaiignore";
+const DEFAULT_IGNORES = "node_modules/\n";
+
+export async function sync(ctx, args) {
+  const [dir, ...extra] = args;
+  if (!dir || extra.length) throw new CliError(EXIT.USAGE, "Usage: syndai sync <dir> [--app A] [--watch]");
+  const root = path.resolve(ctx.io.cwd, dir);
+  if (!fs.statSync(root, { throwIfNoEntry: false })?.isDirectory()) {
+    throw new CliError(EXIT.USAGE, `${dir} is not a folder.`);
+  }
+  const name = path.basename(root);
+  if (!name) throw new CliError(EXIT.USAGE, "Sync a folder with a name, not the filesystem root.");
+  const project = await resolveProject(ctx, { app: ctx.flags.app });
+  // `known`: path -> {size, mtimeMs, hashedAt, sha256 (null: skipped)} as of the last scan.
+  const target = { root, name, project, rules: [], known: new Map() };
+  show(ctx, target, await syncOnce(ctx, target));
+  return ctx.flags.watch ? watchFolder(ctx, target) : EXIT.OK;
+}
+
+// --- one sync ----------------------------------------------------------------------------
+
+async function syncOnce(ctx, target) {
+  const { client } = ctx;
+  const params = { coding_repository_id: target.project.id, name: target.name };
+  scan(target);
+  const files = [...target.known].filter(([, f]) => f.sha256).map(([rel, f]) => ({ path: rel, sha256: f.sha256 }));
+  if (files.length > MANIFEST_MAX) {
+    throw new CliError(
+      EXIT.USAGE,
+      `${target.name} has ${files.length} files; a synced folder holds up to ${MANIFEST_MAX}.`,
+      { next: `list what to leave out in ${path.join(target.root, IGNORE_FILE)}` },
+    );
+  }
+  const manifest = await client.call("compareKnowledgeFolder", {
+    params,
+    body: { files },
+    retry: true, // the same manifest twice is the same answer
+    timeoutMs: 120_000,
+  });
+  const need = manifest.need.filter((rel) => target.known.get(rel)?.sha256);
+  const totals = { indexed: 0, linked: 0, failed: [] };
+  const progress = ctx.io.stderr.isTTY && need.length > BATCH_FILES;
+  let sent = 0;
+  let batch = [];
+  let size = 0;
+  const flush = async () => {
+    const pushed = await client.call("pushKnowledgeFolderFiles", {
+      params,
+      body: { files: batch },
+      retry: true, // a push upserts by path
+      timeoutMs: 120_000,
+    });
+    totals.indexed += pushed.indexed;
+    totals.linked += pushed.linked;
+    totals.failed.push(...pushed.failed);
+    sent += batch.length;
+    if (progress) ctx.ui.progress(ctx.ui.dim(`indexed ${sent} of ${need.length}`));
+    batch = [];
+    size = 0;
+  };
+  for (const rel of need) {
+    // Read again: a file edited since the scan goes with its new hash, so the next manifest
+    // matches it. One gone since is left to the next manifest, which removes it.
+    const file = readFile(target, rel);
+    if (!file?.sha256) continue;
+    const entry = {
+      path: rel,
+      sha256: file.sha256,
+      bytes: file.bytes.length,
+      modified_at: new Date(file.mtimeMs).toISOString(),
+      content: TEXT_SUFFIXES.has(path.extname(rel).toLowerCase()) ? file.bytes.toString("utf8") : null,
+    };
+    const bytes = Buffer.byteLength(JSON.stringify(entry)) + 1;
+    if (batch.length && (batch.length >= BATCH_FILES || size + bytes > BATCH_BYTES)) await flush();
+    batch.push(entry);
+    size += bytes;
+  }
+  if (batch.length) await flush();
+  return {
+    source_id: manifest.source_id,
+    files: files.length,
+    unchanged: files.length - need.length,
+    removed: manifest.removed,
+    ...totals,
+    skipped: [...target.known].filter(([, f]) => !f.sha256).map(([rel]) => ({ path: rel, reason: TOO_BIG })),
+  };
+}
+
+/** Walk the folder into `target.known`, hashing only a file whose size or mtime moved. */
+function scan(target) {
+  target.rules = loadRules(target.root);
+  const before = target.known;
+  target.known = new Map();
+  for (const rel of walk(target.root, target.rules)) {
+    const stat = statFile(target, rel);
+    const last = before.get(rel);
+    if (stat && same(last, stat) && trusted(last)) target.known.set(rel, last);
+    else if (stat) readFile(target, rel);
+  }
+}
+
+const same = (last, stat) => Boolean(last) && last.size === stat.size && last.mtimeMs === stat.mtimeMs;
+
+/** A hash taken soon after the mtime may predate a same-size edit in that clock tick: redo it. */
+const trusted = (last) => !last.sha256 || last.mtimeMs < last.hashedAt - RACY_MS;
+
+/** Whether a stat-only walk differs from the last scan (a file added, removed or touched). */
+function changedOnDisk(target) {
+  const rels = walk(target.root, loadRules(target.root));
+  if (rels.length !== target.known.size) return true;
+  return rels.some((rel) => {
+    const stat = statFile(target, rel);
+    const last = target.known.get(rel);
+    if (!stat || !same(last, stat)) return true;
+    return !trusted(last) && readFile(target, rel)?.sha256 !== last.sha256;
+  });
+}
+
+function statFile(target, rel) {
+  try {
+    const stat = fs.lstatSync(path.join(target.root, ...rel.split("/")));
+    return stat.isFile() ? stat : null;
+  } catch (err) {
+    if (["ENOENT", "EACCES", "EPERM", "ENOTDIR"].includes(err.code)) return null;
+    throw err;
+  }
+}
+
+/**
+ * Read and hash one file into `target.known` and return it with its `bytes`; one over 10 MB is
+ * recorded with `sha256: null` (skipped) and never read. Null when it is gone or unreadable.
+ */
+function readFile(target, rel) {
+  const full = path.join(target.root, ...rel.split("/"));
+  let stat;
+  let bytes;
+  try {
+    stat = fs.lstatSync(full);
+    if (!stat.isFile()) return null;
+    if (stat.size <= FILE_MAX_BYTES) bytes = fs.readFileSync(full);
+  } catch (err) {
+    if (["ENOENT", "EACCES", "EPERM", "EISDIR", "ENOTDIR"].includes(err.code)) {
+      target.known.delete(rel);
+      return null;
+    }
+    throw err;
+  }
+  const small = bytes && bytes.length <= FILE_MAX_BYTES;
+  const file = {
+    size: small ? bytes.length : stat.size,
+    mtimeMs: stat.mtimeMs,
+    hashedAt: Date.now(),
+    sha256: small ? createHash("sha256").update(bytes).digest("hex") : null,
+  };
+  target.known.set(rel, file);
+  return { ...file, bytes };
+}
+
+function show(ctx, target, s) {
+  const lines = [
+    `Synced ${target.name} to ${projectLabel(target.project)}: ` +
+      `${s.indexed} indexed, ${s.linked} linked, ${s.failed.length} failed`,
+  ];
+  if (s.removed) lines.push(`  removed ${s.removed} deleted here`);
+  for (const f of s.failed) lines.push(`  failed ${f.path}: ${f.error}`);
+  for (const f of s.skipped) lines.push(`  skipped ${f.path}: ${f.reason}`);
+  ctx.ui.result({ folder: target.name, dir: target.root, project_id: target.project.id, ...s }, lines.join("\n"));
+}
+
+// --- what is skipped ---------------------------------------------------------------------
+
+function loadRules(root) {
+  let own = "";
+  try {
+    own = fs.readFileSync(path.join(root, IGNORE_FILE), "utf8");
+  } catch {
+    // No .syndaiignore: the defaults alone.
+  }
+  return parseIgnore(DEFAULT_IGNORES + own);
+}
+
+/**
+ * `.syndaiignore`, gitignore-like and kept simple: one pattern a line, `#` comments, `dir/` for
+ * folders only, a pattern with a `/` (or a leading one) anchored at the folder's root and any
+ * other matched against a name at any depth, `*` and `?` within a name, `**` across folders.
+ * No `!` negation.
+ */
+export function parseIgnore(text) {
+  const rules = [];
+  for (const raw of text.split(/\r?\n/)) {
+    let line = raw.trim();
+    if (!line || line.startsWith("#") || line.startsWith("!")) continue;
+    const dirOnly = line.endsWith("/");
+    line = line.replace(/\/+$/, "");
+    const anchored = line.includes("/");
+    line = line.replace(/^\/+/, "");
+    if (line) rules.push({ dirOnly, anchored, regex: globRegex(line) });
+  }
+  return rules;
+}
+
+function globRegex(glob) {
+  let source = "";
+  for (let i = 0; i < glob.length; i++) {
+    const ch = glob[i];
+    if (ch === "*" && glob[i + 1] === "*") {
+      const slash = glob[i + 2] === "/";
+      source += slash ? "(?:.*/)?" : ".*";
+      i += slash ? 2 : 1;
+    } else if (ch === "*") source += "[^/]*";
+    else if (ch === "?") source += "[^/]";
+    else source += ch.replace(/[.+^${}()|[\]\\]/g, "\\$&");
+  }
+  return new RegExp(`^${source}$`);
+}
+
+/** Whether the entry at `rel` (`/`-separated; `isDir` for a folder) is skipped. */
+export function ignored(rules, rel, isDir) {
+  const name = rel.slice(rel.lastIndexOf("/") + 1);
+  if (name.startsWith(".")) return true;
+  return rules.some((r) => (!r.dirOnly || isDir) && r.regex.test(r.anchored ? rel : name));
+}
+
+/** The files to sync, sorted; a skipped folder is never entered. */
+function walk(root, rules) {
+  const files = [];
+  const visit = (dir, prefix) => {
+    let entries;
+    try {
+      entries = fs.readdirSync(dir, { withFileTypes: true });
+    } catch (err) {
+      if (prefix && ["ENOENT", "EACCES", "EPERM", "ENOTDIR"].includes(err.code)) return;
+      throw err;
+    }
+    entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
+    for (const entry of entries) {
+      const rel = prefix ? `${prefix}/${entry.name}` : entry.name;
+      // A Dirent describes the link itself: a symlink is neither, so it is never followed.
+      if (entry.isDirectory()) {
+        if (!ignored(rules, rel, true)) visit(path.join(dir, entry.name), rel);
+      } else if (entry.isFile() && !ignored(rules, rel, false)) {
+        files.push(rel);
+      }
+    }
+  };
+  visit(root, "");
+  return files;
+}
+
+/** A change event for `rel` matters unless it is in a skipped folder or is a skipped file. */
+function relevant(rules, rel) {
+  if (rel === IGNORE_FILE) return true;
+  const parts = rel.split("/");
+  for (let i = 1; i < parts.length; i++) {
+    if (ignored(rules, parts.slice(0, i).join("/"), true)) return false;
+  }
+  return !ignored(rules, rel, false);
+}
+
+// --- --watch -----------------------------------------------------------------------------
+
+async function watchFolder(ctx, target) {
+  const { io, ui } = ctx;
+  const state = { changed: false, wake: null };
+  let watcher = null;
+  const eventsLost = (err) => {
+    watcher?.close();
+    watcher = null;
+    ui.progress(`${ui.yellow("warning:")} file events stopped (${err.code ?? err.message}); checking every 10 s instead.`);
+  };
+  try {
+    watcher = io.watchDir(
+      target.root,
+      (file) => {
+        if (file && !relevant(target.rules, String(file).split(path.sep).join("/"))) return;
+        state.changed = true;
+        state.wake?.();
+      },
+      eventsLost,
+    );
+  } catch (err) {
+    eventsLost(err);
+  }
+  ui.progress(ui.dim(`Watching ${target.root} for changes (Ctrl-C stops).`));
+  try {
+    for (;;) {
+      if (!state.changed && (await nextEvent(io, state)) === "poll" && !changedOnDisk(target)) continue;
+      // Wait until a second passes with no change, so a burst of saves is one sync.
+      do {
+        state.changed = false;
+        await pause(io, DEBOUNCE_MS);
+      } while (state.changed);
+      try {
+        const s = await syncOnce(ctx, target);
+        if (s.indexed || s.linked || s.failed.length || s.removed) show(ctx, target, s);
+      } catch (err) {
+        if (!(err instanceof UnavailableError)) throw err;
+        ui.progress(`${ui.yellow("warning:")} ${explain(err).message} Trying again in 30 s.`);
+        await pause(io, RETRY_MS);
+        state.changed = true;
+      }
+    }
+  } finally {
+    watcher?.close();
+  }
+}
+
+/** "event" on a relevant file event, "poll" after 10 s without one; Ctrl-C interrupts. */
+function nextEvent(io, state) {
+  return new Promise((resolve, reject) => {
+    if (io.signal.aborted) return reject(interrupted());
+    const done = (outcome) => {
+      clearTimeout(timer);
+      io.signal.removeEventListener("abort", onAbort);
+      state.wake = null;
+      if (outcome instanceof Error) reject(outcome);
+      else resolve(outcome);
+    };
+    const onAbort = () => done(interrupted());
+    const timer = setTimeout(() => done("poll"), POLL_MS);
+    io.signal.addEventListener("abort", onAbort, { once: true });
+    state.wake = () => done("event");
+  });
+}
+
+async function pause(io, ms) {
+  if (io.signal.aborted) throw interrupted();
+  try {
+    await io.sleep(ms, io.signal);
+  } catch (err) {
+    if (io.signal.aborted) throw interrupted();
+    throw err;
+  }
+}

package/src/ui.js

@@ -133,6 +133,11 @@ export function openBrowser(url) {
   }
 }
 
+/** `review`'s rule for a hand-off page: open it on a terminal; the caller always prints it. */
+export function openPage({ ui, io }, url) {
+  if (ui.interactive && !ui.json) io.openUrl(url);
+}
+
 /** Show long text through $PAGER on a terminal; print it plainly otherwise. */
 export function page(proc, text) {
   if (!proc.stdout.isTTY) {

package/src/watch.js

@@ -5,14 +5,18 @@
 // cursor. Ctrl-C only detaches: it never sends a cancel. `--logs` also tails the run's SSE
 // events (resuming with Last-Event-ID) next to the loop.
 
-import { ApiError, CliError, EXIT, explain } from "./errors.js";
+import { ApiError, CliError, EXIT, UnavailableError, explain } from "./errors.js";
 import { backoffMs } from "./http.js";
-import { ask, ulid } from "./ui.js";
+import { ask, openPage, ulid } from "./ui.js";
 
 const POLL_WAIT_S = 45;
 const MIN_POLL_GAP_MS = 1000;
 const DEADLINE_GRACE_MS = 2000;
 const TIMED_OUT = Symbol("timed out");
+// A deploy answers 502/503 for a minute or two, longer than one request's retries: the
+// watch keeps polling through it and gives up only after this long without an answer.
+const OUTAGE_MAX_MS = 10 * 60_000;
+const OUTAGE_MAX_WAIT_MS = 30_000;
 
 // Exit code per ChangeState state (docs: syndai.ai/docs/cli, "Exit codes").
 const STATE_EXIT = {
@@ -21,6 +25,7 @@ const STATE_EXIT = {
   ready: EXIT.OK,
   live: EXIT.OK,
   live_unverified: EXIT.UNVERIFIED,
+  unverified: EXIT.UNVERIFIED,
   failed: EXIT.FAILED,
   reverted: EXIT.FAILED,
   cancelled: EXIT.FAILED,
@@ -35,6 +40,7 @@ export function nextFor(state) {
   const id = state.run_id;
   if (state.state === "needs_input") return `syndai answer ${id}`;
   if (state.state === "held") return `syndai review ${id}`;
+  if (state.state === "unverified") return state.pr_url ?? state.run_url ?? null;
   if (state.state === "ready") return `syndai go-live ${id}`;
   if (state.state === "live_unverified") return `syndai watch ${id}`;
   return state.done ? null : `syndai watch ${id}`;
@@ -55,6 +61,8 @@ export function describe(state) {
       return `Held: ${state.held?.message ?? state.held?.reason ?? "needs your review"}`;
     case "ready":
       return `Ready for review${headline}`;
+    case "unverified":
+      return `Delivered, but checks could not be verified${headline}`;
     case "live":
       return `Live: ${state.live_url ?? state.project?.live_url ?? ""}`.trim();
     case "live_unverified":
@@ -68,12 +76,19 @@ export function describe(state) {
 export function finish(ctx, state, { exitCode = exitFor(state), next = nextFor(state) } = {}) {
   const { ui } = ctx;
   const lines = [describe(state)];
+  for (const link of questionLinks(state)) lines.push(`Connect it here: ${link}`);
+  if (state.pr_url) lines.push(ui.dim(state.pr_url));
   if (state.run_url) lines.push(ui.dim(state.run_url));
   ui.result({ ...state, ...(next ? { next } : {}) }, lines.join("\n"));
   if (next && !ui.json) ui.progress(`Next: ${next}`);
   return exitCode;
 }
 
+/** The pages an open question needs (a connection to make), each an absolute portal URL. */
+function questionLinks(state) {
+  return state.state === "needs_input" ? (state.needs_input ?? []).map((q) => q.link).filter(Boolean) : [];
+}
+
 export async function getChange(ctx, runId) {
   return ctx.client.call("changeState", { params: { run_id: runId }, query: { wait: 0 } });
 }
@@ -95,6 +110,8 @@ export async function watchChange(ctx, runId, { timeoutMs = null, untilNot = nul
   let last = null;
   let lastLine = null;
   const answered = new Set(); // a state read before our answer landed must not ask again
+  let outageSince = null;
+  let outageTries = 0;
   try {
     for (;;) {
       const remaining = deadline === null ? Number.POSITIVE_INFINITY : deadline - io.now();
@@ -111,14 +128,24 @@ export async function watchChange(ctx, runId, { timeoutMs = null, untilNot = nul
         });
       } catch (err) {
         if (err === TIMED_OUT) return timedOut(ctx, runId, last);
-        throw err;
+        if (!(err instanceof UnavailableError)) throw err;
+        outageSince ??= io.now();
+        if (io.now() - outageSince >= OUTAGE_MAX_MS) throw err;
+        if (outageTries++ === 0) ui.progress(ui.dim(`· Syndai is unreachable (${err.message}); still watching`));
+        await io.sleep(Math.min(OUTAGE_MAX_WAIT_MS, backoffMs(outageTries + 4)), io.signal);
+        continue;
       }
+      outageSince = null;
+      outageTries = 0;
       const line = describe(state);
       const changed = line !== lastLine || state.cursor !== cursor;
       const final =
         (state.done && state.state !== untilNot) || (state.state === "needs_input" && !ui.interactive);
       if (final) return finish(ctx, state);
-      if (line !== lastLine) ui.progress(`${ui.dim("·")} ${line}`);
+      if (line !== lastLine) {
+        ui.progress(`${ui.dim("·")} ${line}`);
+        if (state.pr_url) ui.progress(ui.dim(state.pr_url));
+      }
       lastLine = line;
       cursor = state.cursor ?? cursor;
       last = state;
@@ -162,6 +189,10 @@ function timedOut(ctx, runId, last) {
 export async function answerInteractively(ctx, runId, questions) {
   for (const question of questions) {
     ctx.ui.progress(ctx.ui.bold(question.summary ?? "A question about your change"));
+    if (question.link) {
+      ctx.ui.progress(`Connect it here: ${question.link}`);
+      openPage(ctx, question.link);
+    }
     const entries = [];
     for (const input of question.inputs ?? []) {
       const entry = await askInput(ctx, input);