quilltap 4.10.0-dev.96 → 4.10.0

package/lib/memories-maintain-command.js

@@ -0,0 +1,238 @@
+'use strict';
+
+/**
+ * `quilltap memories maintain` — run the memory maintenance job by hand:
+ * fold near-duplicates → consolidate → demote → retire → mirror, any subset.
+ *
+ * The CLI's database access is read-only and cannot call a model, so this
+ * resolves the character locally and asks the running server
+ * (POST /api/v1/memories?action=maintain). A dry run is rehearsed in the
+ * server process and prints what each phase would do; a real run is queued
+ * as one MEMORY_MAINTENANCE background job per character.
+ */
+
+const { postMemoriesAction } = require('./memories-server');
+
+// The phase names, the cap's bounds and maxClustersPerRun's bounds are the
+// server's (lib/memory/maintenance/types.ts, lib/schemas/settings.types.ts).
+// The CLI keeps no copy of them: the request goes as typed, and the server's
+// 400 — whose details postMemoriesAction prints — says what it refused.
+
+const RESET = '\x1b[0m';
+const BOLD = '\x1b[1m';
+const DIM = '\x1b[2m';
+const GREEN = '\x1b[32m';
+const YELLOW = '\x1b[33m';
+const CYAN = '\x1b[36m';
+
+function clip(text, n) {
+  const flat = String(text || '').replace(/\s+/g, ' ').trim();
+  return flat.length > n ? flat.slice(0, n - 1) + '…' : flat;
+}
+
+function short(id) {
+  return String(id || '').slice(0, 8);
+}
+
+function bucketLabel(bucket) {
+  if (!bucket) return 'about no one';
+  if (bucket.kind === 'self') return 'about self';
+  if (bucket.kind === 'other') return `about ${bucket.subjectName}`;
+  return 'about no one';
+}
+
+/**
+ * Parse `--phases a,b,…` into a list (trimmed, duplicates dropped). Empty or
+ * absent means "every phase" and returns null. The server checks the names
+ * and always runs them in its own order.
+ */
+function parsePhases(value) {
+  if (value == null || value === '') return null;
+  const asked = [...new Set(String(value).split(',').map((p) => p.trim()).filter(Boolean))];
+  return asked.length === 0 ? null : asked;
+}
+
+/**
+ * Build the `?action=maintain` request body from parsed CLI flags.
+ * `characterId` null means every character the user owns.
+ */
+function buildMaintainBody(characterId, flags) {
+  const body = {};
+  if (characterId) body.characterId = characterId;
+  body.dryRun = flags.dryRun === true;
+  const phases = parsePhases(flags.phases);
+  if (phases) body.phases = phases;
+
+  const settings = {};
+  if (flags.max > 0) settings.maxClustersPerRun = flags.max;
+  if (flags.threshold >= 0) settings.consolidation = { clusterThreshold: flags.threshold };
+  if (flags.cap != null) settings.cap = flags.cap;
+  if (Object.keys(settings).length > 0) body.settings = settings;
+  return body;
+}
+
+// ---------- rendering ----------
+
+/** Pretty-print a ConsolidationReport as cluster → members → digests. */
+function renderConsolidation(report, write) {
+  const stats = report.stats || {};
+  if (report.skippedReason) {
+    write(`  ${YELLOW}Skipped: ${report.skippedReason}${RESET}\n`);
+    return;
+  }
+  const clusters = report.clusters || [];
+  clusters.forEach((cluster, idx) => {
+    write(`\n  ${BOLD}Cluster ${idx + 1}${RESET} ${DIM}(${bucketLabel(cluster.bucket)}, ${cluster.clusterKind}, ${cluster.status})${RESET}\n`);
+    if (cluster.error) write(`    ${YELLOW}${cluster.error}${RESET}\n`);
+    const standalone = new Set(cluster.keepStandalone || []);
+    (cluster.memberIds || []).forEach((id, i) => {
+      const tag = standalone.has(id) ? ` ${DIM}[kept standalone]${RESET}` : '';
+      const content = (cluster.memberContents || [])[i];
+      write(`    ${DIM}-${RESET} ${DIM}${short(id)}${RESET} ${clip(content, 160)}${tag}\n`);
+    });
+    for (const d of cluster.digests || []) {
+      write(`    ${GREEN}=>${RESET} ${CYAN}${d.action === 'update' ? 'update digest' : 'new digest'}${RESET} ` +
+        `${DIM}(imp ${Number(d.importance).toFixed(2)}, from ${(d.memberIds || []).length})${RESET}\n`);
+      write(`       ${clip(d.content, 400)}\n`);
+    }
+    for (const c of cluster.contradictions || []) {
+      write(`    ${YELLOW}contradiction${RESET}: ${clip(c.note, 200)} ${DIM}(${short(c.olderId)} -> ${short(c.newerId)})${RESET}\n`);
+    }
+  });
+  if (clusters.length === 0) write('  (no clusters worth folding)\n');
+  write(`  ${DIM}candidates ${stats.candidates ?? 0}, clusters found ${stats.clustersFound ?? 0}, ` +
+    `attempted ${stats.clustersAttempted ?? 0}, deferred ${stats.clustersDeferred ?? 0}, ` +
+    `${((stats.durationMs ?? 0) / 1000).toFixed(1)}s${stats.budgetExhausted ? ', time budget exhausted' : ''}${RESET}\n`);
+}
+
+function renderFold(fold, write) {
+  write(`  eligible ${fold.eligible ?? 0}, groups ${(fold.groups || []).length}, folded ${fold.folded ?? 0}` +
+    `${fold.deferred ? `, ${fold.deferred} rows deferred` : ''}` +
+    `${fold.groupsDeferred ? `, ${fold.groupsDeferred} groups deferred` : ''}` +
+    `${fold.groupsFailed ? `, ${YELLOW}${fold.groupsFailed} groups failed${RESET}` : ''}\n`);
+  for (const g of fold.groups || []) {
+    const absorbed = (g.absorbedIds || []).map(short).join(', ');
+    write(`    ${short(g.survivorId)} ${GREEN}<=${RESET} ${absorbed} ${DIM}(min cosine ${Number(g.minCosine).toFixed(3)})${RESET}\n`);
+  }
+}
+
+function renderDemote(d, write) {
+  write(`  cap ${d.cap}, hot ${d.hotTotal} (demotable ${d.demotable}, exempt ${d.protected}), ` +
+    `excess ${d.excess}, demoted ${d.demoted}` +
+    `${d.highestScoreDemoted != null ? ` ${DIM}(highest score demoted ${Number(d.highestScoreDemoted).toFixed(3)})${RESET}` : ''}\n`);
+}
+
+function renderRetire(r, write) {
+  write(`  retention ${r.retentionDays} days, retired ${r.retired}\n`);
+}
+
+function renderMirror(m, write) {
+  const subjects = m.subjects || [];
+  write(`  ${subjects.length} subject${subjects.length === 1 ? '' : 's'}${subjects.length ? `: ${subjects.join(', ')}` : ''}` +
+    `, files written ${m.filesWritten ?? 0}${m.deferredToCommit ? ' (deferred to the parent commit)' : ''}\n`);
+}
+
+/** Pretty-print one character's MaintenanceReport: phase lines, then per-phase detail. */
+function renderMaintenanceReport(report, write = (s) => process.stdout.write(s)) {
+  const who = report.characterName || short(report.characterId);
+  write(`${BOLD}Maintenance ${report.dryRun ? 'dry run' : 'report'}${RESET}${who ? ` for ${who}` : ''}\n`);
+  if (report.skippedReason) {
+    write(`${YELLOW}Skipped: ${report.skippedReason}${RESET}\n`);
+    return;
+  }
+  for (const p of report.phases || []) {
+    const status = p.status === 'ran' ? `${GREEN}ran${RESET}    ` : `${DIM}skipped${RESET}`;
+    const why = p.skippedBecause ? ` ${DIM}(${p.skippedBecause})${RESET}` : '';
+    write(`  ${p.phase.padEnd(22)} ${status}${why} ${DIM}${p.durationMs ?? 0}ms${RESET}\n`);
+  }
+  if (report.fold) { write(`\n${BOLD}Fold near-duplicates${RESET}\n`); renderFold(report.fold, write); }
+  if (report.consolidation) { write(`\n${BOLD}Consolidate${RESET}\n`); renderConsolidation(report.consolidation, write); }
+  if (report.demote) { write(`\n${BOLD}Demote${RESET}\n`); renderDemote(report.demote, write); }
+  if (report.retire) { write(`\n${BOLD}Retire${RESET}\n`); renderRetire(report.retire, write); }
+  if (report.mirror) { write(`\n${BOLD}Mirror${RESET}\n`); renderMirror(report.mirror, write); }
+  if (report.durationMs != null) write(`\n${DIM}${(report.durationMs / 1000).toFixed(1)}s${RESET}\n`);
+}
+
+/** One table row per character for a user-wide dry run. */
+function renderReportsTable(reports, write = (s) => process.stdout.write(s)) {
+  const dash = '-';
+  const rows = (reports || []).map((r) => {
+    if (r.skippedReason) {
+      return [r.characterName || short(r.characterId), `skipped: ${r.skippedReason}`, '', '', '', ''];
+    }
+    const ran = (r.phases || []).filter((p) => p.status === 'ran').map((p) => p.phase).join(',') || 'none';
+    return [
+      r.characterName || short(r.characterId),
+      ran,
+      r.fold ? `${(r.fold.groups || []).length}/${r.fold.folded ?? 0}` : dash,
+      r.demote ? `${r.demote.demotable}/${r.demote.cap} -${r.demote.demoted}` : dash,
+      r.retire ? String(r.retire.retired) : dash,
+      r.mirror ? String((r.mirror.subjects || []).length) : dash,
+    ];
+  });
+  const header = ['character', 'phases ran', 'fold grp/folded', 'demote demotable/cap -n', 'retire', 'mirror subj'];
+  const widths = header.map((h, i) => Math.max(h.length, ...rows.map((row) => String(row[i]).length)));
+  const line = (cells) => cells.map((c, i) => String(c).padEnd(widths[i])).join('  ').trimEnd() + '\n';
+  write(`${BOLD}Maintenance dry run across ${rows.length} character${rows.length === 1 ? '' : 's'}${RESET}\n`);
+  write(line(header));
+  write(line(widths.map((w) => '-'.repeat(w))));
+  for (const row of rows) write(line(row));
+  if (rows.length === 0) write('(no characters)\n');
+}
+
+// ---------- command ----------
+
+/**
+ * @param {object} flags parsed memories flags
+ * @param {(flags: object) => Promise<string>} resolveCharacterId resolves --character to a UUID
+ */
+async function cmdMaintain(flags, resolveCharacterId) {
+  if (!flags.character) {
+    throw new Error("memories maintain requires --character <name|id|all> ('all' for every character).");
+  }
+  let characterId = null;
+  if (flags.character !== 'all') {
+    characterId = await resolveCharacterId(flags);
+    if (!/^[0-9a-fA-F-]{36}$/.test(characterId)) {
+      throw new Error(`could not resolve --character to a UUID (got "${characterId}").`);
+    }
+  }
+  const body = buildMaintainBody(characterId, flags);
+
+  const payload = await postMemoriesAction(flags, 'maintain', body, {
+    why: 'Maintenance runs inside the server (the model, the embeddings and the only database writer live there).',
+    announce: flags.json ? undefined : (url) => process.stderr.write(
+      `${BOLD}${body.dryRun ? 'Rehearsing' : 'Queuing'} maintenance${RESET}` +
+      `${body.phases ? ` [${body.phases.join(',')}]` : ''} via ${DIM}${url}${RESET}\n`
+    ),
+  });
+
+  if (flags.json) {
+    process.stdout.write(JSON.stringify(payload, null, 2) + '\n');
+    return;
+  }
+
+  if (body.dryRun) {
+    if (payload && payload.report) renderMaintenanceReport(payload.report);
+    else renderReportsTable(payload && payload.reports);
+    return;
+  }
+
+  const jobIds = (payload && payload.jobIds) || (payload && payload.jobId ? [payload.jobId] : []);
+  if (jobIds.length === 0) {
+    console.log('No maintenance jobs were queued (no characters to maintain).');
+    return;
+  }
+  console.log(`Maintenance queued as ${jobIds.length} job${jobIds.length === 1 ? '' : 's'}:`);
+  for (const id of jobIds) console.log(`  ${id}`);
+  console.log("They run in the background; watch logs/combined.log or 'quilltap memories status'.");
+}
+
+module.exports = {
+  cmdMaintain,
+  parsePhases,
+  buildMaintainBody,
+  renderMaintenanceReport,
+  renderReportsTable,
+  renderConsolidation,
+};

package/lib/memories-server.js

@@ -0,0 +1,86 @@
+'use strict';
+
+/**
+ * The one way a `quilltap memories` verb asks the running server to do what
+ * the CLI cannot: anything that calls a model, computes an embedding or
+ * writes. The CLI's own database access is read-only; `maintain` and
+ * `validate --repair` resolve what they can locally and POST the rest to
+ * `/api/v1/memories?action=<action>`.
+ *
+ * The port comes from `resolveServerPort` (lock-helpers): an explicit
+ * `--port`, else the port recorded in the `--instance` / `--data-dir`
+ * instance's lock, else 3000.
+ */
+
+const { resolveServerPort } = require('./lock-helpers');
+
+const DIM = '\x1b[2m';
+const RESET = '\x1b[0m';
+
+/** Resolve the server port for a memories verb, printing the lock note (if any) to stderr. */
+function resolveMemoriesServerPort(flags, { quiet = false } = {}) {
+  const target = resolveServerPort({
+    port: flags.port,
+    instance: flags.instance || undefined,
+    dataDir: flags.dataDir || undefined,
+  });
+  if (target.note && !quiet) process.stderr.write(`${DIM}${target.note}${RESET}\n`);
+  return target;
+}
+
+/**
+ * POST `body` to `/api/v1/memories?action=<action>` and return the parsed
+ * payload. Throws an Error with a readable message on a connection failure
+ * or a non-2xx response.
+ *
+ * @param {object} flags            parsed memories flags (port / instance / dataDir)
+ * @param {string} action           the `?action=` name
+ * @param {object} body             JSON request body
+ * @param {object} [opts]
+ * @param {string} [opts.why]       one line on why the server is needed
+ * @param {(msg: string) => void} [opts.announce] called with the URL before the request
+ * @param {typeof fetch} [opts.fetchImpl] for tests
+ */
+async function postMemoriesAction(flags, action, body, opts = {}) {
+  const { why, announce, fetchImpl = fetch } = opts;
+  const target = resolveMemoriesServerPort(flags);
+  const url = `http://localhost:${target.port}/api/v1/memories?action=${encodeURIComponent(action)}`;
+  if (announce) announce(url);
+
+  let res;
+  try {
+    res = await fetchImpl(url, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(body),
+    });
+  } catch (err) {
+    throw new Error(
+      `Could not reach the Quilltap server at http://localhost:${target.port}: ${err.message}\n` +
+      (why ? `${why}\n` : '') +
+      target.hint
+    );
+  }
+
+  let payload = null;
+  try {
+    const text = await res.text();
+    payload = text ? JSON.parse(text) : null;
+  } catch { /* leave payload null */ }
+
+  if (!res.ok) {
+    let msg = (payload && payload.error) || `HTTP ${res.status}`;
+    if (payload && Array.isArray(payload.details) && payload.details.length > 0) {
+      msg += ': ' + payload.details
+        .map((d) => `${Array.isArray(d.path) && d.path.length ? d.path.join('.') + ' ' : ''}${d.message || ''}`.trim())
+        .join('; ');
+    }
+    if (res.status === 400 && /unknown action|invalid action/i.test(msg)) {
+      msg += ` (is the server older than this CLI? '${action}' needs a matching Quilltap server)`;
+    }
+    throw new Error(msg);
+  }
+  return payload;
+}
+
+module.exports = { postMemoriesAction, resolveMemoriesServerPort };

package/lib/memory-diff-command.js

@@ -1,5 +1,7 @@
 'use strict';
 
+const { resolveServerPort } = require('./lock-helpers');
+
 /**
  * `quilltap memory-diff <chatId>` — dump existing memories for a chat and
  * stream a dry-run re-extraction from the running server, writing both to
@@ -45,7 +47,8 @@ Options:
   -d, --data-dir <path>      Override data directory (instance root)
   -i, --instance <name>      Use a registered instance (see 'quilltap instances')
       --passphrase <pass>    Decrypt .dbkey if peppered
-      --port <number>        Server port for API calls (default: 3000)
+      --port <number>        Server port for API calls. Default: the port recorded in the
+                             --instance/--data-dir server's lock file, else 3000
       --out <dir>            Output directory (default: cwd)
       --concurrency <number> Parallel turns to process (default: 4, max: 32).
                              Cloud cheap-LLMs can handle 8–16; keep this low
@@ -65,7 +68,7 @@ function parseFlags(args) {
     dataDir: '',
     instance: '',
     passphrase: '',
-    port: 3000,
+    port: undefined,
     out: process.cwd(),
     concurrency: 4,
     help: false,
@@ -256,6 +259,16 @@ async function memoryDiffCommand(args) {
 
   const chatId = positional[0];
 
+  let target;
+  try {
+    target = resolveServerPort({ port: flags.port, instance: flags.instance, dataDir: flags.dataDir });
+  } catch (err) {
+    console.error(`Error: ${err.message}`);
+    process.exit(1);
+  }
+  const port = target.port;
+  if (target.note) process.stderr.write(`${DIM}${target.note}${RESET}\n`);
+
   if (!fs.existsSync(flags.out)) {
     try {
       fs.mkdirSync(flags.out, { recursive: true });
@@ -276,7 +289,7 @@ async function memoryDiffCommand(args) {
 
   // -------- 2. Stream dry-run re-extraction from the server ----------------
   const url =
-    `http://localhost:${flags.port}/api/v1/chats/${encodeURIComponent(chatId)}` +
+    `http://localhost:${port}/api/v1/chats/${encodeURIComponent(chatId)}` +
     `?action=extract-memories-dry-run&concurrency=${flags.concurrency}`;
   process.stderr.write(
     `${BOLD}Streaming re-extraction${RESET} (concurrency ${CYAN}${flags.concurrency}${RESET}) from ${DIM}${url}${RESET}\n`
@@ -286,8 +299,8 @@ async function memoryDiffCommand(args) {
   try {
     res = await fetch(url, { method: 'POST' });
   } catch (err) {
-    console.error(`${RED}Could not reach Quilltap server at http://localhost:${flags.port}: ${err.message}${RESET}`);
-    console.error('Start the server (npm run dev) or pass --port to match a non-default port.');
+    console.error(`${RED}Could not reach Quilltap server at http://localhost:${port}: ${err.message}${RESET}`);
+    console.error(target.hint);
     process.exit(1);
   }
 
@@ -328,6 +341,15 @@ async function memoryDiffCommand(args) {
           `${GREEN}${event.candidatesAdded}${RESET} candidate(s)\n`
         );
         break;
+      case 'fold':
+        // The fold passes run after every turn, over the history a regenerate
+        // would rebuild; their candidates arrive with turnIndex null.
+        process.stderr.write(
+          `  ${CYAN}[${event.pass}]${RESET} ${event.slices} slice(s): ` +
+          `${GREEN}${event.candidatesAdded}${RESET} candidate(s)` +
+          `${event.skippedReason ? ` ${DIM}(${event.skippedReason})${RESET}` : ''}\n`
+        );
+        break;
       case 'turn-error':
         process.stderr.write(
           `  ${RED}[${event.index + 1}/${turnCount}] FAILED${RESET}: ${event.error}\n`
@@ -368,4 +390,4 @@ async function memoryDiffCommand(args) {
   if (fatal) process.exit(2);
 }
 
-module.exports = { memoryDiffCommand };
+module.exports = { memoryDiffCommand, parseFlags };

package/lib/migrations-commands.js

@@ -43,6 +43,19 @@ function parseFlags(args) {
 
 // ---------- migration list parsing ----------
 
+/**
+ * A migration's id is either a string literal on the object (`id: 'foo-v1'`)
+ * or a module constant (`id: MIGRATION_ID` beside `const MIGRATION_ID = 'foo-v1'`).
+ * Missing the constant form drops those migrations from "source", which then
+ * report as retired on every instance that ran them.
+ */
+function extractMigrationId(fileContent) {
+  const constMatch = fileContent.match(/^\s*const\s+MIGRATION_ID\s*(?::\s*\w+\s*)?=\s*['"]([^'"]+)['"]/m);
+  if (constMatch && /^\s*id:\s*MIGRATION_ID\b/m.test(fileContent)) return constMatch[1];
+  const literalMatch = fileContent.match(/^\s*id:\s*['"]([^'"]+)['"]/m);
+  return literalMatch ? literalMatch[1] : null;
+}
+
 // Reads the registered migration list from migrations/scripts/index.ts and resolves each
 // entry's real `id:` value by reading the imported source file. Heuristic camelCase→kebab
 // conversion is unreliable (some migrations omit `-v1`, others use `-v2`), so we always
@@ -95,11 +108,11 @@ function extractMigrationsFromSource() {
     const filePath = path.join(scriptsDir, info.filePath + '.ts');
     if (!fs.existsSync(filePath)) continue;
     const fileContent = fs.readFileSync(filePath, 'utf-8');
-    const idMatch = fileContent.match(/^\s*id:\s*['"]([^'"]+)['"]/m);
-    if (!idMatch) continue;
+    const id = extractMigrationId(fileContent);
+    if (!id) continue;
     migrations.push({
-      id: idMatch[1],
-      description: info.comment || idMatch[1],
+      id,
+      description: info.comment || id,
     });
   }
 
@@ -321,4 +334,5 @@ async function migrationsCommand(args) {
 
 module.exports = {
   migrationsCommand,
+  extractMigrationId,
 };