quilltap 4.10.0-dev.9 → 4.10.0-dev.92

package/lib/completion/zsh.template

@@ -37,6 +37,7 @@ _quilltap() {
   subcommands=(
     'db:Query encrypted databases'
     'docs:Inspect, read, and export document mounts'
+    'sync:Mirror a database-backed store to a directory'
     'themes:Manage theme bundles'
     'instances:Register or inspect named Quilltap instances'
     'memories:Search, browse, and graph memories'
@@ -80,6 +81,9 @@ _quilltap_subcommand() {
     docs)
       _quilltap_docs
       ;;
+    sync)
+      _quilltap_sync
+      ;;
     themes)
       _quilltap_themes
       ;;
@@ -314,6 +318,38 @@ _quilltap_docs() {
   esac
 }
 
+# `quilltap sync <store> <path>` — a store, then a local directory.
+_quilltap_sync() {
+  local -a sync_opts
+  sync_opts=(
+    '(-i --instance)'{-i,--instance}'[Registered instance name]:instance:_quilltap_instance_names'
+    '(-d --data-dir)'{-d,--data-dir}'[Data directory]:directory:_directories'
+    '--passphrase[Database passphrase]:passphrase:'
+    '(-p --port)'{-p,--port}'[Server port]:port:'
+    '--json[Machine-readable plan and results]'
+    '--dry-run[Plan and print; change nothing]'
+    '--direction[Which side may change]:direction:(both to-disk to-store)'
+    '--prefer[How to resolve a conflict]:prefer:(newer store disk)'
+    '--no-delete[Never propagate a deletion]'
+    '--no-manifest[Ignore .quilltap-sync.json]'
+    '(-h --help)'{-h,--help}'[Show help]'
+  )
+
+  _arguments -C $sync_opts \
+    '1: :->store' \
+    '2: :->target' \
+    '*: :'
+
+  case "$state" in
+    store)
+      _quilltap_mount_names
+      ;;
+    target)
+      _directories
+      ;;
+  esac
+}
+
 # Which docs positionals name a document store, and which name a local path.
 # `move`/`copy`/`link` take <srcMount> <srcPath> <dstMount> <dstPath>, so a
 # store is wanted at both 2 and 4; everything else that takes a store takes it

package/lib/db-commands.js

@@ -16,6 +16,7 @@ const {
   resolveProject,
 } = require('./db-helpers');
 const { getLockStatus } = require('./lock-helpers');
+const { decodeText } = require('./text-codec');
 
 // Tables grouped by domain for `db schema` (with-no-arg) overview, and for
 // DB-routing when a verb names a specific table. Keep this list short — it's
@@ -477,6 +478,11 @@ function cmdMessages(args, ctx) {
     params.push(last);
     const rows = db.prepare(sql).all(...params).reverse(); // oldest first
 
+    // `content` may be a brotli BLOB (see lib/text-codec.js). Decode before
+    // anything prints or JSON-serializes it — a raw BLOB prints as binary and
+    // JSON.stringify turns it into a byte array.
+    for (const r of rows) r.content = decodeText(r.content);
+
     if (json) return printJson({ chat, total: totalRow.n, returned: rows.length, messages: rows });
 
     console.log(`Chat: ${chat.title} (${chat.id}) — showing ${rows.length} of ${totalRow.n} matching messages`);
@@ -557,6 +563,13 @@ function cmdMessage(args, ctx) {
   try {
     const row = db.prepare('SELECT * FROM chat_messages WHERE id = ?').get(id);
     if (!row) throw new Error(`No chat_message with id ${id}`);
+
+    // The four large text columns may be brotli BLOBs (see lib/text-codec.js).
+    // Decode before anything prints or JSON-serializes them.
+    for (const col of ['content', 'opaqueContent', 'description', 'context']) {
+      if (col in row) row[col] = decodeText(row[col]);
+    }
+
     if (json) return printJson(row);
 
     printRecord(`Message ${row.id}`, {
@@ -596,6 +609,13 @@ function cmdLog(args, ctx) {
   try {
     const row = db.prepare('SELECT * FROM llm_logs WHERE id = ?').get(id);
     if (!row) throw new Error(`No llm_log with id ${id}`);
+
+    // request/response may be brotli BLOBs (see lib/text-codec.js). Decode
+    // before anything reads, prints or JSON-serializes them — a raw BLOB
+    // would print as binary and JSON.stringify as a byte array.
+    row.request = decodeText(row.request);
+    row.response = decodeText(row.response);
+
     if (json) return printJson(row);
 
     let finishReason = null;

package/lib/db-helpers.js

@@ -2,6 +2,7 @@
 
 const path = require('path');
 const fs = require('fs');
+const { registerTextCodecFunction } = require('./text-codec');
 
 const CTRL_C = String.fromCharCode(3);
 const CTRL_D = String.fromCharCode(4);
@@ -201,6 +202,17 @@ function openEncryptedDb(dbPath, pepper, { readonly = true, friendlyName = 'data
       'The database may be encrypted with a different key, or the .dbkey file may be missing.');
   }
 
+  // Compressed text columns (llm_logs.request/response, chat_messages.content
+  // and friends) are BLOBs; qt_text() lets raw SQL and the repl read inside
+  // them. It is also REQUIRED for any --write that touches chat_messages: the
+  // message search triggers call it, so a connection without it fails the
+  // write loudly rather than letting the index drift.
+  try {
+    registerTextCodecFunction(db);
+  } catch {
+    // An old better-sqlite3 without db.function must not block a read.
+  }
+
   return db;
 }
 

package/lib/native-modules.js

@@ -62,6 +62,84 @@ function betterSqlite3NeedsRebuild() {
   }
 }
 
+// Locate an executable installed by a dependency, given the package directory
+// that needs it. Checks the package's own `.bin` first, then the `.bin` beside
+// it (the hoisted case, which is the usual one). Returns null when absent.
+function findBinFor(pkgDir, tool) {
+  const fs = require('fs');
+  const names = process.platform === 'win32' ? [`${tool}.cmd`, `${tool}.exe`, tool] : [tool];
+  const dirs = [
+    path.join(pkgDir, 'node_modules', '.bin'),  // nested install
+    path.join(pkgDir, '..', '.bin'),            // hoisted alongside the package
+  ];
+  for (const dir of dirs) {
+    for (const name of names) {
+      const candidate = path.join(dir, name);
+      if (fs.existsSync(candidate)) return candidate;
+    }
+  }
+  return null;
+}
+
+// Rebuild ONE native package in place, addressed by its directory.
+//
+// Deliberately does NOT go through `npm rebuild <name>`, which fails this job
+// two different ways:
+//
+//   1. npm refuses install scripts for any package not listed in the root
+//      package.json `allowScripts` map (keyed `name@version`). The root
+//      SQLCipher copy is installed under the ALIAS `better-sqlite3`, which is
+//      not a key there, so the rebuild dies with EALLOWSCRIPTS.
+//   2. Asking for the real name (`better-sqlite3-multiple-ciphers`) at the root
+//      instead resolves a phantom directory: npm reports "rebuilt dependencies
+//      successfully" and the binding on disk is untouched. A success message is
+//      worse than an error.
+//
+// Running the package's own build chain (`prebuild-install || node-gyp rebuild`,
+// exactly what its `install` script does) in its own directory sidesteps both:
+// no name resolution, no npm script policy. `prebuild-install` downloads the
+// prebuilt binary for the running ABI and needs network; node-gyp compiles and
+// needs a toolchain.
+//
+// Returns { ok, reason }. Never throws. Verifies the result rather than trusting
+// the exit code — see (2): the whole failure mode here is a rebuild that claims
+// to have worked.
+function rebuildNativePackage(pkgDir, bindingPath) {
+  const fs = require('fs');
+  if (!fs.existsSync(pkgDir)) return { ok: false, reason: `no package at ${pkgDir}` };
+
+  const before = fs.existsSync(bindingPath) ? readCompiledAbi(bindingPath) : null;
+  const attempts = [];
+
+  const prebuild = findBinFor(pkgDir, 'prebuild-install');
+  if (prebuild) attempts.push({ label: 'prebuild-install', cmd: `"${prebuild}"` });
+
+  const nodeGyp = findBinFor(pkgDir, 'node-gyp');
+  if (nodeGyp) attempts.push({ label: 'node-gyp rebuild', cmd: `"${nodeGyp}" rebuild --release` });
+
+  if (attempts.length === 0) {
+    return { ok: false, reason: 'neither prebuild-install nor node-gyp is installed' };
+  }
+
+  const failures = [];
+  for (const attempt of attempts) {
+    try {
+      execSync(attempt.cmd, { cwd: pkgDir, stdio: 'inherit' });
+    } catch (err) {
+      failures.push(`${attempt.label}: ${err.message.split('\n')[0]}`);
+      continue;
+    }
+    // Trust nothing but the binary. A tool can exit 0 having done nothing.
+    const after = fs.existsSync(bindingPath) ? readCompiledAbi(bindingPath) : null;
+    if (after === process.versions.modules) return { ok: true, reason: attempt.label };
+    failures.push(
+      `${attempt.label}: exited 0 but the binding is still ${after ?? 'missing'} ` +
+      `(wanted ${process.versions.modules}${before ? `, was ${before}` : ''})`,
+    );
+  }
+  return { ok: false, reason: failures.join('; ') };
+}
+
 // Rebuild the named native modules against the current Node ABI. Prints a
 // friendly notice rather than throwing; returns true on success, false on
 // failure. Backfills node-pty's spawn-helper afterward.
@@ -96,7 +174,27 @@ function ensureDatabaseNativeModule() {
   } catch {
     return true; // detection hiccup — let the real load be the source of truth
   }
-  return rebuildModules(['better-sqlite3-multiple-ciphers']);
+
+  const bindingPath = betterSqlite3BindingPath();
+  if (!bindingPath) {
+    console.error('  Warning: could not locate the SQLCipher binding to rebuild.');
+    return false;
+  }
+  // build/Release/better_sqlite3.node → the package directory above it.
+  const pkgDir = path.resolve(path.dirname(bindingPath), '..', '..');
+
+  console.log(`  Rebuilding the SQLCipher binding for Node.js ${process.version}...`);
+  const result = rebuildNativePackage(pkgDir, bindingPath);
+  if (result.ok) {
+    console.log(`  Done (${result.reason}).`);
+    console.log('');
+    return true;
+  }
+  console.error('');
+  console.error(`  Warning: failed to rebuild the SQLCipher binding — ${result.reason}`);
+  console.error(`  Try running: (cd ${pkgDir} && ./node_modules/.bin/prebuild-install)`);
+  console.error('');
+  return false;
 }
 
 // node-pty needs a `spawn-helper` executable beside the pty.node it loads, or
@@ -194,7 +292,10 @@ function ensureNativeModules() {
 module.exports = {
   resolveModuleDir,
   readCompiledAbi,
+  betterSqlite3BindingPath,
   betterSqlite3NeedsRebuild,
+  findBinFor,
+  rebuildNativePackage,
   ensureDatabaseNativeModule,
   ensureNativeModules,
   reconcileNodePtySpawnHelper,

package/lib/sync-command.js

@@ -0,0 +1,323 @@
+'use strict';
+
+/**
+ * `quilltap sync <store> <path>` — keep a database-backed document store and a
+ * directory on disk in step with each other.
+ *
+ * This file is a thin client. Every decision the sync makes — which side wins,
+ * what counts as a deletion, when to refuse — is made by the engine in the
+ * server (`lib/mount-index/sync/`), and every flag is validated by the route's
+ * own schema, which is the single source of truth. The CLI resolves the store
+ * name to a UUID (the one thing it opens the database for, read-only), posts
+ * the request, and prints what came back.
+ *
+ * The engine lives in the server rather than here because the sync writes
+ * through the store's own chokepoints — `linkDocumentContent`, the folder-row
+ * helper, the hard-link fan-out, the post-write re-chunk — and those are
+ * TypeScript in `lib/`, unreachable from this plain-JS package. A direct
+ * SQLite writer would be a second copy of all of them, and a lock-gated one
+ * could not re-chunk at all. `docs write` on a database store already requires
+ * the server for exactly this reason; `deconvert` already takes a server-local
+ * target path. This is that shape, not a new one.
+ *
+ * `<path>` is therefore resolved on the SERVER. Running under Docker, it must
+ * sit inside a bind mount — `quilltap docs docker-mounts` plans those.
+ *
+ * @module sync-command
+ */
+
+const path = require('path');
+const os = require('os');
+const {
+  resolveDataDirAndPassphrase,
+  printDefaultInstanceHint,
+  loadDbKey,
+  openMountIndexDb,
+  UUID_RE,
+} = require('./db-helpers');
+const { isQtapUri, parseQtapUri } = require('./qtap-uri');
+const { formatActionLines, formatSummary, exitCodeFor } = require('./sync-report');
+
+const RESET = '\x1b[0m';
+const DIM = '\x1b[2m';
+const YELLOW = '\x1b[33m';
+
+function printSyncHelp() {
+  console.log(`
+Quilltap Document Store Sync
+
+Usage: quilltap sync <store|qtap://store/> <path> [options]
+
+Mirrors a database-backed document store and a directory on disk in both
+directions. The two sides are compared by SHA-256 first and by modification
+time second: equal bytes with unequal clocks are re-stamped, not re-copied.
+Whichever side changed is copied to the other; when both changed since the
+last run it is reported as a conflict and left alone.
+
+The directory is created if it is missing. A document store is never created.
+
+Options:
+  --dry-run                 Plan and print; change nothing on either side
+  --direction <which>       both (default), to-disk, or to-store
+  --prefer <which>          newer (default), store, or disk — resolves conflicts
+  --no-delete               Never propagate a deletion to the other side
+  --no-manifest             Ignore .quilltap-sync.json (first-run rules every time)
+  --json                    Machine-readable plan and results on stdout
+  -p, --port <number>       Server port (default: 3000)
+  -d, --data-dir <path>     Override data directory
+  -i, --instance <name>     Use a registered instance
+  --passphrase <pass>       Decrypt .dbkey if peppered
+  -h, --help                Show this help
+
+Exit codes: 0 clean, 1 an error or a failed action, 2 an unresolved conflict.
+--dry-run uses the same codes, so a script can gate on a clean plan.
+
+Files and folders whose names begin with a dot are INVISIBLE to the sync in
+both directions — never copied, never deleted, on either side. The one
+exception is .quilltap-sync.json, the record the verb keeps of what the last
+run left; it lives in the directory and never enters the store.
+
+A binary's description travels beside it as <file>.description.md. Editing
+that file changes the caption in the store; deleting it clears the caption.
+A text document's description is not synced.
+
+The server must be running: a database-backed store's writes go through it,
+as they already do for 'quilltap docs write'. <path> is resolved on the
+server — under Docker it must sit inside a bind mount (see
+'quilltap docs docker-mounts').
+
+Examples:
+  quilltap sync Lore ~/Documents/lore --dry-run
+  quilltap sync Lore ~/Documents/lore
+  quilltap sync Lore ~/Documents/lore --prefer disk
+  quilltap sync qtap://Lore/ ~/Documents/lore --direction to-disk
+`);
+}
+
+function parseFlags(args) {
+  const flags = {
+    dataDir: '',
+    instance: '',
+    passphrase: '',
+    port: 3000,
+    json: false,
+    dryRun: false,
+    direction: 'both',
+    prefer: 'newer',
+    noDelete: false,
+    noManifest: false,
+    help: false,
+  };
+  const positional = [];
+
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i];
+    switch (arg) {
+      case '-d': case '--data-dir':    flags.dataDir = args[++i] || ''; break;
+      case '-i': case '--instance':    flags.instance = args[++i] || ''; break;
+      case '--passphrase':             flags.passphrase = args[++i] || ''; break;
+      case '-p': case '--port':        flags.port = parseInt(args[++i], 10) || 3000; break;
+      case '--json':                   flags.json = true; break;
+      case '--dry-run':                flags.dryRun = true; break;
+      case '--direction':              flags.direction = args[++i] || ''; break;
+      case '--prefer':                 flags.prefer = args[++i] || ''; break;
+      case '--no-delete':              flags.noDelete = true; break;
+      case '--no-manifest':            flags.noManifest = true; break;
+      case '-h': case '--help':        flags.help = true; break;
+      default:
+        if (arg.startsWith('-')) {
+          console.error(`Unknown option: ${arg}`);
+          process.exit(1);
+        }
+        positional.push(arg);
+    }
+  }
+  return { flags, positional };
+}
+
+/** `~/x` → an absolute path. The server sees only what we send it. */
+function expandPath(input) {
+  let expanded = input;
+  if (expanded === '~') expanded = os.homedir();
+  else if (expanded.startsWith('~/')) expanded = path.join(os.homedir(), expanded.slice(2));
+  return path.resolve(expanded);
+}
+
+/**
+ * A store name or UUID, or a `qtap://store/` URI with an empty path. This is
+ * the only thing the verb opens the database for, and it opens it read-only.
+ */
+function resolveStoreSpec(spec) {
+  if (!isQtapUri(spec)) return spec;
+  const parsed = parseQtapUri(spec);
+  if (parsed.scope !== 'document_store') {
+    throw new Error(
+      'sync addresses document stores only; qtap://project/… and qtap://general/… are not CLI-addressable.'
+    );
+  }
+  if (parsed.path) {
+    throw new Error(`sync takes a whole store, not a path inside one: ${spec}`);
+  }
+  if (!parsed.mountPoint || parsed.mountPoint.toLowerCase() === 'self') {
+    throw new Error('"self" requires a character context and is not resolvable from the CLI.');
+  }
+  return parsed.mountPoint;
+}
+
+/** Name-first, UUID fallback — the same resolution the `docs` verbs use. */
+function requireMount(db, spec) {
+  if (UUID_RE.test(spec)) {
+    const row = db.prepare('SELECT * FROM doc_mount_points WHERE id = ?').get(spec);
+    if (!row) {
+      console.error(`No document store found with id ${spec}`);
+      process.exit(1);
+    }
+    return row;
+  }
+  const rows = db.prepare(
+    `SELECT * FROM doc_mount_points WHERE LOWER(name) = LOWER(?) ORDER BY name COLLATE NOCASE`
+  ).all(spec);
+  if (rows.length === 0) {
+    console.error(`No document store found with name "${spec}"`);
+    process.exit(1);
+  }
+  if (rows.length > 1) {
+    console.error(`Ambiguous store name "${spec}" matches multiple stores:`);
+    for (const r of rows) console.error(`  ${r.id}  ${r.name}  (${r.mountType})`);
+    console.error('Pass the UUID instead.');
+    process.exit(1);
+  }
+  return rows[0];
+}
+
+function isConnectionRefused(err) {
+  if (!err) return false;
+  const code = err.cause && err.cause.code ? err.cause.code : err.code;
+  return code === 'ECONNREFUSED' || code === 'ENOTFOUND' ||
+         code === 'EHOSTUNREACH' || code === 'ECONNRESET';
+}
+
+async function syncCommand(args) {
+  const { flags, positional } = parseFlags(args);
+  if (flags.help || positional.length === 0) {
+    printSyncHelp();
+    process.exit(flags.help ? 0 : 1);
+  }
+
+  const [storeSpec, targetSpec] = positional;
+  if (!targetSpec) {
+    console.error('Usage: quilltap sync <store> <path> [options]');
+    console.error("Run 'quilltap sync --help' for the full list of options.");
+    process.exit(1);
+  }
+  if (positional.length > 2) {
+    console.error(`sync takes one store and one path; got ${positional.length} arguments.`);
+    process.exit(1);
+  }
+
+  let storeName;
+  try {
+    storeName = resolveStoreSpec(storeSpec);
+  } catch (err) {
+    console.error(`Error: ${err.message}`);
+    process.exit(1);
+  }
+
+  const targetPath = expandPath(targetSpec);
+
+  // Resolve the store from the local database — read-only, and the only thing
+  // the CLI opens it for.
+  const resolved = resolveDataDirAndPassphrase({
+    dataDir: flags.dataDir,
+    instance: flags.instance,
+    passphrase: flags.passphrase,
+  });
+  printDefaultInstanceHint(resolved);
+  const pepper = await loadDbKey(resolved.dataDir, resolved.passphrase);
+  const db = openMountIndexDb(resolved.dataDir, pepper, { readonly: true });
+  let mount;
+  try {
+    mount = requireMount(db, storeName);
+  } finally {
+    db.close();
+  }
+
+  if (mount.mountType !== 'database') {
+    console.error(
+      `"${mount.name}" is a ${mount.mountType} store — it already IS a directory` +
+      (mount.basePath ? ` (${mount.basePath})` : '') + '.'
+    );
+    console.error('sync mirrors database-backed stores only.');
+    process.exit(1);
+  }
+
+  const url =
+    `http://localhost:${flags.port}/api/v1/mount-points/${encodeURIComponent(mount.id)}?action=sync`;
+
+  // The route's schema is the single source of truth for these; the CLI does
+  // not re-validate, so a bad --direction is refused by the server with the
+  // server's own wording.
+  const body = {
+    targetPath,
+    dryRun: flags.dryRun,
+    direction: flags.direction,
+    prefer: flags.prefer,
+    propagateDeletes: !flags.noDelete,
+    useManifest: !flags.noManifest,
+  };
+
+  let res;
+  try {
+    res = await fetch(url, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(body),
+    });
+  } catch (err) {
+    if (isConnectionRefused(err)) {
+      console.error(
+        `Cannot sync database-backed store "${mount.name}" without the Quilltap server.`
+      );
+      console.error('Start the server (`quilltap`) or pass --port to match a non-default port.');
+      process.exit(1);
+    }
+    throw err;
+  }
+
+  const payload = await res.json().catch(() => null);
+  if (!res.ok) {
+    const message = payload && payload.error ? payload.error : `HTTP ${res.status}`;
+    console.error(`Error: ${message}`);
+    process.exit(1);
+  }
+
+  const report = payload && payload.data !== undefined ? payload.data : payload;
+
+  if (flags.json) {
+    console.log(JSON.stringify(report, null, 2));
+    process.exit(exitCodeFor(report.summary));
+  }
+
+  const colour = Boolean(process.stdout.isTTY);
+  for (const line of formatActionLines(report.actions, colour)) {
+    console.log(line);
+  }
+
+  // Advisory text goes to stderr so stdout stays greppable.
+  for (const warning of report.warnings || []) {
+    console.error(colour ? `${YELLOW}warning:${RESET} ${warning}` : `warning: ${warning}`);
+  }
+  const summary = formatSummary(report.summary, report.elapsedMs, report.dryRun);
+  console.error(colour ? `${DIM}${summary}${RESET}` : summary);
+  if (report.summary.conflicts > 0) {
+    console.error(
+      colour
+        ? `${DIM}Re-run with --prefer store or --prefer disk to resolve the conflicts.${RESET}`
+        : 'Re-run with --prefer store or --prefer disk to resolve the conflicts.'
+    );
+  }
+
+  process.exit(exitCodeFor(report.summary));
+}
+
+module.exports = { syncCommand, printSyncHelp, expandPath, resolveStoreSpec };