quilltap 4.10.0-dev.18 → 4.10.0-dev.184

package/lib/db-commands.js

@@ -16,6 +16,7 @@ const {
   resolveProject,
 } = require('./db-helpers');
 const { getLockStatus } = require('./lock-helpers');
+const { decodeText } = require('./text-codec');
 
 // Tables grouped by domain for `db schema` (with-no-arg) overview, and for
 // DB-routing when a verb names a specific table. Keep this list short — it's
@@ -477,6 +478,11 @@ function cmdMessages(args, ctx) {
     params.push(last);
     const rows = db.prepare(sql).all(...params).reverse(); // oldest first
 
+    // `content` may be a brotli BLOB (see lib/text-codec.js). Decode before
+    // anything prints or JSON-serializes it — a raw BLOB prints as binary and
+    // JSON.stringify turns it into a byte array.
+    for (const r of rows) r.content = decodeText(r.content);
+
     if (json) return printJson({ chat, total: totalRow.n, returned: rows.length, messages: rows });
 
     console.log(`Chat: ${chat.title} (${chat.id}) — showing ${rows.length} of ${totalRow.n} matching messages`);
@@ -557,6 +563,13 @@ function cmdMessage(args, ctx) {
   try {
     const row = db.prepare('SELECT * FROM chat_messages WHERE id = ?').get(id);
     if (!row) throw new Error(`No chat_message with id ${id}`);
+
+    // The four large text columns may be brotli BLOBs (see lib/text-codec.js).
+    // Decode before anything prints or JSON-serializes them.
+    for (const col of ['content', 'opaqueContent', 'description', 'context']) {
+      if (col in row) row[col] = decodeText(row[col]);
+    }
+
     if (json) return printJson(row);
 
     printRecord(`Message ${row.id}`, {
@@ -596,6 +609,13 @@ function cmdLog(args, ctx) {
   try {
     const row = db.prepare('SELECT * FROM llm_logs WHERE id = ?').get(id);
     if (!row) throw new Error(`No llm_log with id ${id}`);
+
+    // request/response may be brotli BLOBs (see lib/text-codec.js). Decode
+    // before anything reads, prints or JSON-serializes them — a raw BLOB
+    // would print as binary and JSON.stringify as a byte array.
+    row.request = decodeText(row.request);
+    row.response = decodeText(row.response);
+
     if (json) return printJson(row);
 
     let finishReason = null;
@@ -638,10 +658,20 @@ function cmdMemories(args, ctx) {
   const { flags } = parseSubArgs(args);
   const json = asBool(flags.json);
   const limit = asInt(flags.limit, 50);
-  if (!flags.character) throw new Error('Usage: quilltap db memories --character <id|name> [--about <id|name>] [--source AUTO|MANUAL] [--limit N]');
+  if (!flags.character) throw new Error('Usage: quilltap db memories --character <id|name> [--about <id|name>] [--source AUTO|MANUAL|CONSOLIDATED] [--tier hot|cold] [--limit N]');
+  const tier = flags.tier === undefined ? null : String(flags.tier).toLowerCase();
+  if (tier !== null && tier !== 'hot' && tier !== 'cold') {
+    throw new Error(`--tier must be hot or cold (got '${flags.tier === true ? '' : flags.tier}')`);
+  }
 
   const db = ctx.openMain();
   try {
+    // Instances that predate the memory tiers have no `tier` column: every
+    // row reads as hot there.
+    let hasTier = false;
+    try {
+      hasTier = db.prepare("PRAGMA table_info('memories')").all().some((c) => c.name === 'tier');
+    } catch { /* treat as absent */ }
     const holder = resolveCharacter(db, String(flags.character), ctx.openMounts);
     const conditions = ['characterId = ?'];
     const params = [holder.id];
@@ -654,8 +684,13 @@ function cmdMemories(args, ctx) {
       conditions.push('source = ?');
       params.push(String(flags.source).toUpperCase());
     }
+    if (tier) {
+      if (hasTier) conditions.push(tier === 'hot' ? "COALESCE(tier, 'hot') = 'hot'" : "tier = 'cold'");
+      else if (tier === 'cold') conditions.push('0');
+    }
+    const tierSelect = hasTier ? "COALESCE(tier, 'hot') AS tier" : "'hot' AS tier";
     const rows = db.prepare(`
-      SELECT id, createdAt, source, importance, reinforcementCount,
+      SELECT id, createdAt, source, ${tierSelect}, importance, reinforcementCount,
              aboutCharacterId, chatId, summary
       FROM memories WHERE ${conditions.join(' AND ')}
       ORDER BY createdAt DESC LIMIT ?
@@ -665,11 +700,12 @@ function cmdMemories(args, ctx) {
 
     if (json) return printJson({ holder, total: totalRow.n, returned: rows.length, memories: rows });
 
-    console.log(`Memories held by ${holder.name} (${holder.id}) — ${rows.length} of ${totalRow.n}`);
+    console.log(`Memories held by ${holder.name} (${holder.id})${tier ? ` [${tier}]` : ''} — ${rows.length} of ${totalRow.n}`);
     printTable(rows.map(r => ({
       createdAt: r.createdAt,
       id: r.id,
       src: r.source,
+      tier: r.tier,
       imp: r.importance,
       rein: r.reinforcementCount,
       about: r.aboutCharacterId === holder.id ? 'self' : (r.aboutCharacterId || ''),

package/lib/db-helpers.js

@@ -2,6 +2,7 @@
 
 const path = require('path');
 const fs = require('fs');
+const { registerTextCodecFunction } = require('./text-codec');
 
 const CTRL_C = String.fromCharCode(3);
 const CTRL_D = String.fromCharCode(4);
@@ -201,6 +202,17 @@ function openEncryptedDb(dbPath, pepper, { readonly = true, friendlyName = 'data
       'The database may be encrypted with a different key, or the .dbkey file may be missing.');
   }
 
+  // Compressed text columns (llm_logs.request/response, chat_messages.content
+  // and friends) are BLOBs; qt_text() lets raw SQL and the repl read inside
+  // them. It is also REQUIRED for any --write that touches chat_messages: the
+  // message search triggers call it, so a connection without it fails the
+  // write loudly rather than letting the index drift.
+  try {
+    registerTextCodecFunction(db);
+  } catch {
+    // An old better-sqlite3 without db.function must not block a read.
+  }
+
   return db;
 }
 

package/lib/graph-integrity.js

@@ -3,6 +3,11 @@
 // Shared graph-integrity scanner used by `memories status` (per-holder
 // rollup) and `memories validate` (read-only dangling-edge check).
 //
+// `scanDanglingEdges` covers `relatedMemoryIds`; `scanDigestIntegrity`
+// covers the consolidation links (`consolidatedFrom`, `supersededById`) and
+// rows with no embedding — the same four things the server's graph repair
+// (`POST /api/v1/memories?action=repair`) finds and fixes.
+//
 // "Dangling edges" are UUIDs in `memories.relatedMemoryIds` that no longer
 // resolve to a row in the `memories` table. Cross-character links are
 // legitimate — a memory's links may point at memories owned by other holders
@@ -34,10 +39,7 @@ function scanDanglingEdges(db, opts = {}) {
   // The valid set is *every* memory ID in the table. Computed once even
   // when restricted to a single holder, because cross-character links are
   // legitimate and we don't want them counted as dangling.
-  const allIds = new Set();
-  for (const row of db.prepare('SELECT id FROM memories').all()) {
-    allIds.add(row.id);
-  }
+  const allIds = opts.allIds || loadMemoryIds(db);
 
   const scanRows = characterId
     ? db.prepare('SELECT id, characterId, relatedMemoryIds FROM memories WHERE characterId = ?').all(characterId)
@@ -102,4 +104,105 @@ function scanDanglingEdges(db, opts = {}) {
   return result;
 }
 
-module.exports = { scanDanglingEdges };
+/** Every memory id in the table — the universe a link may legitimately name. */
+function loadMemoryIds(db) {
+  const allIds = new Set();
+  for (const row of db.prepare('SELECT id FROM memories').all()) {
+    allIds.add(row.id);
+  }
+  return allIds;
+}
+
+/** The memories table's column names (old instances lack the tier-era columns). */
+function memoryColumns(db) {
+  try {
+    return new Set(db.prepare("PRAGMA table_info('memories')").all().map((c) => c.name));
+  } catch {
+    return new Set();
+  }
+}
+
+/**
+ * Scan the consolidation links and embeddings for one holder (or every row).
+ *
+ * - dangling `consolidatedFrom`: member ids a digest names that no longer exist
+ * - dangling `supersededById`: members pointing at a digest that is gone
+ * - missing embeddings: rows with `embedding IS NULL` (not recallable by
+ *   meaning until re-embedded; a warning, not graph damage)
+ *
+ * Columns absent on very old instances read as nothing to scan.
+ *
+ * @param {import('better-sqlite3').Database} db  Open readonly DB handle.
+ * @param {object} [opts]
+ * @param {string} [opts.characterId]  Restrict the scan to one holder.
+ * @param {boolean} [opts.includePairs] Return the offending rows (for --list).
+ * @param {Set<string>} [opts.allIds]  Reuse a precomputed id universe.
+ * @returns {{
+ *   consolidatedFrom: { links: number, digests: number },
+ *   supersededBy: number,
+ *   missingEmbeddings: number,
+ *   consolidatedFromPairs?: Array<{ digestId: string, characterId: string, memberIds: string[] }>,
+ *   supersededByPairs?: Array<{ memberId: string, characterId: string, digestId: string }>
+ * }}
+ */
+function scanDigestIntegrity(db, opts = {}) {
+  const { characterId, includePairs = false } = opts;
+  const cols = opts.columns || memoryColumns(db);
+  const allIds = opts.allIds || loadMemoryIds(db);
+  const where = characterId ? ' AND characterId = ?' : '';
+  const params = characterId ? [characterId] : [];
+
+  const result = {
+    consolidatedFrom: { links: 0, digests: 0 },
+    supersededBy: 0,
+    missingEmbeddings: 0,
+  };
+  if (includePairs) {
+    result.consolidatedFromPairs = [];
+    result.supersededByPairs = [];
+  }
+
+  if (cols.has('consolidatedFrom')) {
+    const rows = db.prepare(
+      `SELECT id, characterId, consolidatedFrom FROM memories
+       WHERE consolidatedFrom IS NOT NULL AND consolidatedFrom != '[]'${where}`
+    ).all(...params);
+    for (const row of rows) {
+      let arr;
+      try { arr = JSON.parse(row.consolidatedFrom || '[]'); } catch { arr = []; }
+      if (!Array.isArray(arr)) continue;
+      const missing = arr.filter((id) => !allIds.has(id));
+      if (missing.length === 0) continue;
+      result.consolidatedFrom.links += missing.length;
+      result.consolidatedFrom.digests++;
+      if (includePairs) {
+        result.consolidatedFromPairs.push({ digestId: row.id, characterId: row.characterId, memberIds: missing });
+      }
+    }
+  }
+
+  if (cols.has('supersededById')) {
+    const rows = db.prepare(
+      `SELECT id, characterId, supersededById FROM memories
+       WHERE supersededById IS NOT NULL${where}`
+    ).all(...params);
+    for (const row of rows) {
+      if (allIds.has(row.supersededById)) continue;
+      result.supersededBy++;
+      if (includePairs) {
+        result.supersededByPairs.push({ memberId: row.id, characterId: row.characterId, digestId: row.supersededById });
+      }
+    }
+  }
+
+  if (cols.has('embedding')) {
+    const row = db.prepare(
+      `SELECT COUNT(*) AS n FROM memories WHERE embedding IS NULL${where}`
+    ).get(...params);
+    result.missingEmbeddings = row ? row.n || 0 : 0;
+  }
+
+  return result;
+}
+
+module.exports = { scanDanglingEdges, scanDigestIntegrity, loadMemoryIds, memoryColumns };

package/lib/lock-helpers.js

@@ -110,6 +110,70 @@ function getLockStatus(dataDir) {
   };
 }
 
+// ============================================================================
+// Server port resolution
+//
+// CLI tools that talk to a RUNNING server over HTTP (memory-diff, recall-replay,
+// anchor-probe, memories consolidate) need its port. The server records the
+// port it listens on in the instance lock, so naming an instance (or a data
+// dir) is enough to find it. An explicit --port always wins.
+// ============================================================================
+
+const DEFAULT_SERVER_PORT = 3000;
+
+/**
+ * Decide which port to call.
+ *
+ * @param {{ port?: number, instance?: string, dataDir?: string }} opts
+ *   `port` is the explicit --port (undefined when not given).
+ * @returns {{ port: number, source: 'flag'|'lock'|'default', note: string|null,
+ *            hint: string }}
+ *   `note` is a one-line message to print when the lock supplied the port;
+ *   `hint` is the connection-failure advice.
+ * @throws if both instance and dataDir are given, or the instance is unknown.
+ */
+function resolveServerPort({ port, instance, dataDir } = {}) {
+  const baseHint = 'Start the server (npm run dev) or pass --port to match a non-default port.';
+  if (port !== undefined && port !== null) {
+    return { port, source: 'flag', note: null, hint: baseHint };
+  }
+  if (!instance && !dataDir) {
+    return { port: DEFAULT_SERVER_PORT, source: 'default', note: null, hint: baseHint };
+  }
+
+  // Lazy require: db-helpers is heavier and nothing else here needs it.
+  const { resolveDataDirAndPassphrase } = require('./db-helpers');
+  const resolved = resolveDataDirAndPassphrase({ instance, dataDir });
+  const label = resolved.instanceName || resolved.dataDir;
+  const status = getLockStatus(resolved.dataDir);
+  const lockPort = status.lock && Number.isInteger(status.lock.port) &&
+    status.lock.port >= 1 && status.lock.port <= 65535
+    ? status.lock.port
+    : null;
+
+  if (status.state === 'active' && lockPort !== null) {
+    return {
+      port: lockPort,
+      source: 'lock',
+      note: `Using port ${lockPort} from the ${label} instance lock`,
+      hint: baseHint,
+    };
+  }
+
+  let why;
+  if (status.state === 'active') why = 'its lock records no port (server predates port recording)';
+  else if (status.state === 'absent') why = 'it has no instance lock, so no server appears to be running';
+  else if (status.state === 'stale') why = `its lock is stale (${status.reason})`;
+  else if (status.state === 'suspect') why = `its lock looks suspect (${status.reason})`;
+  else why = 'its lock file is unreadable';
+  return {
+    port: DEFAULT_SERVER_PORT,
+    source: 'default',
+    note: null,
+    hint: `${baseHint}\nNo port found for the ${label} instance: ${why}; fell back to ${DEFAULT_SERVER_PORT}.`,
+  };
+}
+
 // ============================================================================
 // Write-lock acquire / release
 //
@@ -354,4 +418,6 @@ module.exports = {
   acquireWriteLock,
   releaseWriteLock,
   detectEnvironmentType,
+  resolveServerPort,
+  DEFAULT_SERVER_PORT,
 };