threadnote 1.4.2 → 1.4.3

package/config/post-update-migrations.json

@@ -18,6 +18,21 @@
         "If Threadnote reported any original files were still being processed, rerun the printed threadnote forget <uri> command later."
       ],
       "requiresLegacyHandoffs": true
+    },
+    {
+      "id": "ov-semantic-poison-hotfix-2734",
+      "introducedIn": "1.4.3",
+      "title": "Patch OpenViking against the semantic-queue poison loop (#2734)",
+      "description": [
+        "OpenViking 0.4.x can deadlock its semantic queue: a memory file reindexed with mode=semantic_and_vectors enqueues a directory-level semantic message whose URI is a file; the processor lists it, fails, and the message re-enqueues forever (it is AGFS-persisted, so it survives a server restart).",
+        "This patches the installed OpenViking to skip non-directory/missing memory URIs (upstream fix PR #2735) and restarts the server so any stuck message drains on the next dequeue.",
+        "It is a no-op when the installed OpenViking already includes the fix, a backup of the original file is kept, and the patched file is compile-checked before it is written."
+      ],
+      "commandArgs": ["repair-semantic-queue", "--apply"],
+      "instructions": [
+        "If the semantic queue was stuck it should now drain. Verify with: ov observer queue",
+        "This is a temporary local patch; it is superseded automatically once Threadnote pins an OpenViking release that includes the fix. Re-run manually any time with: threadnote repair-semantic-queue --apply"
+      ]
     }
   ]
 }

package/dist/threadnote.cjs

@@ -15009,6 +15009,199 @@ function parsePackageManager(value) {
   throw new Error(`Invalid package manager: ${value}`);
 }
 
+// src/semantic_queue_repair.ts
+var import_node_fs8 = require("node:fs");
+var import_promises14 = require("node:fs/promises");
+var import_node_os7 = require("node:os");
+var import_node_path15 = require("node:path");
+var HOTFIX_MARKER = "THREADNOTE-HOTFIX-2734";
+var ENV_OVERRIDE = "THREADNOTE_OPENVIKING_SEMANTIC_PROCESSOR";
+function patchSemanticProcessorSource(source) {
+  if (source.includes(HOTFIX_MARKER)) {
+    return { status: "already-fixed" };
+  }
+  const lines = source.split("\n");
+  const defIndex = lines.findIndex((line) => /^\s*async def _process_memory_directory\b/.test(line));
+  if (defIndex < 0) {
+    return { status: "no-anchor" };
+  }
+  const lsPattern = /^(\s+)entries = await viking_fs\.ls\(dir_uri\b/;
+  let lsIndex = -1;
+  let lsIndent = "";
+  for (let i = defIndex + 1; i < lines.length; i += 1) {
+    const match = lsPattern.exec(lines[i]);
+    if (match) {
+      lsIndex = i;
+      lsIndent = match[1];
+      break;
+    }
+    if (/^\s*async def \w/.test(lines[i])) {
+      break;
+    }
+  }
+  if (lsIndex < 0) {
+    return { status: "no-anchor" };
+  }
+  if (lines.slice(defIndex, lsIndex).some((line) => line.includes("viking_fs.stat(dir_uri"))) {
+    return { status: "already-fixed" };
+  }
+  let tryIndex = -1;
+  let base = "";
+  for (let j = lsIndex - 1; j > defIndex; j -= 1) {
+    const text = lines[j];
+    if (text.trim() === "" || text.trimStart().startsWith("#")) {
+      continue;
+    }
+    const match = /^(\s*)try:\s*$/.exec(text);
+    if (match && match[1].length < lsIndent.length) {
+      tryIndex = j;
+      base = match[1];
+    }
+    break;
+  }
+  if (tryIndex < 0) {
+    return { status: "no-anchor" };
+  }
+  const unit = " ".repeat(lsIndent.length - base.length);
+  const at = (level, text) => `${base}${unit.repeat(level)}${text}`;
+  const guard = [
+    at(0, `# ${HOTFIX_MARKER}: skip non-directory / missing memory URIs so a memory file`),
+    at(0, "# reindexed with mode=semantic_and_vectors cannot poison the semantic queue."),
+    at(0, "# Temporary local hot-fix; superseded once OpenViking ships upstream PR #2735."),
+    at(0, "try:"),
+    at(1, "_tn_dir_stat = await viking_fs.stat(dir_uri, ctx=ctx)"),
+    at(0, "except Exception as _tn_err:"),
+    at(1, 'if isinstance(_tn_err, FileNotFoundError) or "not found" in str(_tn_err).lower():'),
+    at(2, "_mark_done()"),
+    at(2, "return"),
+    at(1, "_tn_dir_stat = None"),
+    at(0, 'if _tn_dir_stat is not None and not _tn_dir_stat.get("isDir", _tn_dir_stat.get("is_dir", False)):'),
+    at(1, "_mark_done()"),
+    at(1, "return")
+  ];
+  const patched = [...lines.slice(0, tryIndex), ...guard, ...lines.slice(tryIndex)].join("\n");
+  return { status: "patched", source: patched };
+}
+async function semanticProcessorCandidates(root) {
+  const lib = (0, import_node_path15.join)(root, "lib");
+  if (!(0, import_node_fs8.existsSync)(lib)) {
+    return [];
+  }
+  let entries;
+  try {
+    entries = await (0, import_promises14.readdir)(lib);
+  } catch {
+    return [];
+  }
+  return entries.filter((name) => name.startsWith("python")).map((name) => (0, import_node_path15.join)(lib, name, "site-packages", "openviking", "storage", "queuefs", "semantic_processor.py"));
+}
+async function locateSemanticProcessorPath() {
+  const override = process.env[ENV_OVERRIDE]?.trim();
+  if (override) {
+    const resolved = expandPath(override);
+    if ((0, import_node_fs8.existsSync)(resolved)) {
+      return resolved;
+    }
+  }
+  const roots = [];
+  const server = await findOpenVikingServer();
+  if (server) {
+    roots.push((0, import_node_path15.dirname)((0, import_node_path15.dirname)(server)));
+  }
+  roots.push((0, import_node_path15.join)((0, import_node_os7.homedir)(), ".local", "share", "uv", "tools", "openviking"));
+  roots.push((0, import_node_path15.join)((0, import_node_os7.homedir)(), ".local", "pipx", "venvs", "openviking"));
+  for (const root of roots) {
+    for (const candidate of await semanticProcessorCandidates(root)) {
+      if ((0, import_node_fs8.existsSync)(candidate)) {
+        return candidate;
+      }
+    }
+  }
+  if (server) {
+    const venvPython = (0, import_node_path15.join)((0, import_node_path15.dirname)(server), process.platform === "win32" ? "python.exe" : "python3");
+    if ((0, import_node_fs8.existsSync)(venvPython)) {
+      const result = await runCommand(
+        venvPython,
+        [
+          "-c",
+          'import openviking, os; print(os.path.join(os.path.dirname(openviking.__file__), "storage", "queuefs", "semantic_processor.py"))'
+        ],
+        { allowFailure: true }
+      );
+      const path2 = result.stdout.trim();
+      if (result.exitCode === 0 && path2 && (0, import_node_fs8.existsSync)(path2)) {
+        return path2;
+      }
+    }
+  }
+  return void 0;
+}
+async function assertPatchedSourceCompiles(source) {
+  const python = await findExecutable(["python3", "python"]);
+  if (!python) {
+    console.warn("WARN python3 not found; skipping compile validation of the patched OpenViking source.");
+    return;
+  }
+  const dir = await (0, import_promises14.mkdtemp)((0, import_node_path15.join)((0, import_node_os7.tmpdir)(), "threadnote-ov-patch-"));
+  const file = (0, import_node_path15.join)(dir, "semantic_processor_patched.py");
+  try {
+    await (0, import_promises14.writeFile)(file, source, "utf8");
+    const result = await runCommand(python, ["-m", "py_compile", file], { allowFailure: true });
+    if (result.exitCode !== 0) {
+      throw new Error(`patched OpenViking source failed to compile: ${result.stderr.trim() || result.stdout.trim()}`);
+    }
+  } finally {
+    await (0, import_promises14.rm)(dir, { force: true, recursive: true });
+  }
+}
+async function runRepairSemanticQueue(config, options) {
+  const apply = options.apply === true && options.dryRun !== true;
+  const path2 = await locateSemanticProcessorPath();
+  if (!path2) {
+    console.error(
+      `Could not locate the installed OpenViking semantic_processor.py. Set ${ENV_OVERRIDE}=/path/to/openviking/storage/queuefs/semantic_processor.py and retry.`
+    );
+    process.exitCode = 1;
+    return;
+  }
+  console.log(`OpenViking semantic processor: ${path2}`);
+  let original;
+  try {
+    original = await (0, import_promises14.readFile)(path2, "utf8");
+  } catch (err) {
+    console.error(`Could not read ${path2}: ${errorMessage(err)}`);
+    process.exitCode = 1;
+    return;
+  }
+  const result = patchSemanticProcessorSource(original);
+  if (result.status === "already-fixed") {
+    console.log("OpenViking already guards non-directory memory URIs (#2734/#2735); nothing to patch.");
+    return;
+  }
+  if (result.status === "no-anchor") {
+    console.warn(
+      "WARN Could not find the _process_memory_directory ls(dir_uri) anchor; this OpenViking layout is unexpected, so no changes were made."
+    );
+    return;
+  }
+  if (!apply) {
+    console.log("Dry run: would patch OpenViking to skip non-directory/missing memory URIs, then restart the server.");
+    console.log("Re-run with --apply to perform it.");
+    return;
+  }
+  await assertPatchedSourceCompiles(result.source);
+  const backup = `${path2}.threadnote-bak`;
+  if (!(0, import_node_fs8.existsSync)(backup)) {
+    await (0, import_promises14.copyFile)(path2, backup);
+  }
+  await (0, import_promises14.writeFile)(path2, result.source, "utf8");
+  console.log(`Patched OpenViking semantic processor (backup: ${backup}).`);
+  console.log("Restarting the OpenViking server so the patch loads and any stuck message drains...");
+  await runStop(config, {});
+  await runStart(config, {});
+  console.log("Done. A stuck semantic message drains on the next dequeue; verify with: ov observer queue");
+}
+
 // src/version_command.ts
 async function runVersion(config, options) {
   const currentVersion = await currentPackageVersion();
@@ -15052,9 +15245,9 @@ function printWhatsNew(whatsNew) {
 // src/manager.ts
 var import_node_http2 = require("node:http");
 var import_node_crypto2 = require("node:crypto");
-var import_promises14 = require("node:fs/promises");
-var import_node_os7 = require("node:os");
-var import_node_path15 = require("node:path");
+var import_promises15 = require("node:fs/promises");
+var import_node_os8 = require("node:os");
+var import_node_path16 = require("node:path");
 var STATIC_FILES = {
   "/": { contentType: "text/html; charset=utf-8", path: "index.html" },
   "/index.html": { contentType: "text/html; charset=utf-8", path: "index.html" },
@@ -15125,18 +15318,18 @@ async function readManagedMemory(config, uri) {
   if (!path2) {
     throw new Error(`Manager can only read current-user memory URIs: ${uri}`);
   }
-  const [content, pathStat] = await Promise.all([(0, import_promises14.readFile)(path2, "utf8"), (0, import_promises14.stat)(path2)]);
-  const relativePath = (0, import_node_path15.relative)(localMemoriesRoot(config), path2).split(import_node_path15.sep).join("/");
+  const [content, pathStat] = await Promise.all([(0, import_promises15.readFile)(path2, "utf8"), (0, import_promises15.stat)(path2)]);
+  const relativePath = (0, import_node_path16.relative)(localMemoriesRoot(config), path2).split(import_node_path16.sep).join("/");
   const record = parseMemoryDocument(uri, content);
   return {
     content,
     node: {
       isDir: false,
       isShared: isInSharedNamespace(config, uri),
-      isSystem: isSystemMemoryName(path2.split(import_node_path15.sep).at(-1) ?? ""),
+      isSystem: isSystemMemoryName(path2.split(import_node_path16.sep).at(-1) ?? ""),
       metadata: record?.metadata,
       modTime: pathStat.mtime.toISOString(),
-      name: path2.split(import_node_path15.sep).at(-1) ?? uri,
+      name: path2.split(import_node_path16.sep).at(-1) ?? uri,
       relativePath,
       sharedTeam: sharedTeamNameForUri(config, uri),
       size: pathStat.size,
@@ -15424,7 +15617,7 @@ async function handleRequest(context, request, response) {
 }
 async function serveStatic(context, url, response) {
   const file = STATIC_FILES[url.pathname] ?? STATIC_FILES["/"];
-  const content = await (0, import_promises14.readFile)((0, import_node_path15.join)(toolRoot(), file.root ?? "manager", file.path));
+  const content = await (0, import_promises15.readFile)((0, import_node_path16.join)(toolRoot(), file.root ?? "manager", file.path));
   const headers = { "content-type": file.contentType };
   if (file.root !== "docs") {
     headers["cache-control"] = "no-store";
@@ -15433,11 +15626,11 @@ async function serveStatic(context, url, response) {
   response.end(content);
 }
 async function readTree(config, path2, uri, relativePath, options = {}) {
-  const pathStat = await (0, import_promises14.stat)(path2);
+  const pathStat = await (0, import_promises15.stat)(path2);
   const name = relativePath ? relativePath.split("/").at(-1) ?? relativePath : options.rootName ?? "memories";
   const isDir = pathStat.isDirectory();
   if (!isDir) {
-    const record = options.parseMemoryDocuments === false ? void 0 : parseMemoryDocument(uri, await (0, import_promises14.readFile)(path2, "utf8").catch(() => ""));
+    const record = options.parseMemoryDocuments === false ? void 0 : parseMemoryDocument(uri, await (0, import_promises15.readFile)(path2, "utf8").catch(() => ""));
     return {
       isDir: false,
       isShared: isInSharedNamespace(config, uri),
@@ -15451,13 +15644,13 @@ async function readTree(config, path2, uri, relativePath, options = {}) {
       uri
     };
   }
-  const entries = await (0, import_promises14.readdir)(path2, { withFileTypes: true });
+  const entries = await (0, import_promises15.readdir)(path2, { withFileTypes: true });
   const children = await Promise.all(
     entries.sort(
       (left, right) => Number(right.isDirectory()) - Number(left.isDirectory()) || left.name.localeCompare(right.name)
     ).map((entry) => {
       const childRelative = relativePath ? `${relativePath}/${entry.name}` : entry.name;
-      return readTree(config, (0, import_node_path15.join)(path2, entry.name), `${uri}/${entry.name}`, childRelative, options);
+      return readTree(config, (0, import_node_path16.join)(path2, entry.name), `${uri}/${entry.name}`, childRelative, options);
     })
   );
   return {
@@ -15631,27 +15824,27 @@ async function removeManagedFolder(config, uri) {
   if (!path2) {
     throw new Error(`Manager can only remove current-user memory folders: ${uri}`);
   }
-  const pathStat = await (0, import_promises14.stat)(path2);
+  const pathStat = await (0, import_promises15.stat)(path2);
   if (!pathStat.isDirectory()) {
     throw new Error(`Not a folder: ${uri}`);
   }
-  const relativePath = (0, import_node_path15.relative)(localMemoriesRoot(config), path2);
-  if (!relativePath || relativePath.startsWith("..") || relativePath.split(import_node_path15.sep).includes("..")) {
+  const relativePath = (0, import_node_path16.relative)(localMemoriesRoot(config), path2);
+  if (!relativePath || relativePath.startsWith("..") || relativePath.split(import_node_path16.sep).includes("..")) {
     throw new Error("Refusing to remove a folder outside the memories tree.");
   }
   const fileUris = await fileUrisUnderFolder(config, path2);
   for (const fileUri of fileUris) {
     await runForget(config, fileUri, {});
   }
-  await (0, import_promises14.rm)(path2, { force: true, recursive: true });
+  await (0, import_promises15.rm)(path2, { force: true, recursive: true });
   console.log(`Removed folder: ${uri}`);
   console.log(`Forgot ${fileUris.length} file${fileUris.length === 1 ? "" : "s"}.`);
 }
 async function fileUrisUnderFolder(config, folderPath) {
-  const entries = await (0, import_promises14.readdir)(folderPath, { withFileTypes: true });
+  const entries = await (0, import_promises15.readdir)(folderPath, { withFileTypes: true });
   const uris = [];
   for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) {
-    const path2 = (0, import_node_path15.join)(folderPath, entry.name);
+    const path2 = (0, import_node_path16.join)(folderPath, entry.name);
     if (entry.isDirectory()) {
       uris.push(...await fileUrisUnderFolder(config, path2));
     } else if (entry.isFile()) {
@@ -15749,11 +15942,11 @@ async function runConsolidationAgent(agent, sources) {
     throw new Error(`${agent} executable was not found.`);
   }
   const prompt = consolidationPrompt(sources);
-  const stagingDir = await (0, import_promises14.mkdtemp)((0, import_node_path15.join)((0, import_node_os7.tmpdir)(), "threadnote-consolidate-"));
-  const promptPath = (0, import_node_path15.join)(stagingDir, "prompt.txt");
+  const stagingDir = await (0, import_promises15.mkdtemp)((0, import_node_path16.join)((0, import_node_os8.tmpdir)(), "threadnote-consolidate-"));
+  const promptPath = (0, import_node_path16.join)(stagingDir, "prompt.txt");
   try {
-    await (0, import_promises14.writeFile)(promptPath, prompt, { encoding: "utf8", mode: 384 });
-    await (0, import_promises14.chmod)(promptPath, 384);
+    await (0, import_promises15.writeFile)(promptPath, prompt, { encoding: "utf8", mode: 384 });
+    await (0, import_promises15.chmod)(promptPath, 384);
     const script = consolidationAgentScript(agent, executable);
     const result = await runCommand("sh", ["-lc", script, "threadnote-consolidate", promptPath], {
       allowFailure: true,
@@ -15769,7 +15962,7 @@ async function runConsolidationAgent(agent, sources) {
     }
     return draft;
   } finally {
-    await (0, import_promises14.rm)(stagingDir, { force: true, recursive: true });
+    await (0, import_promises15.rm)(stagingDir, { force: true, recursive: true });
   }
 }
 function consolidationAgentScript(agent, executable) {
@@ -15907,10 +16100,10 @@ function memoryDirectoryUri2(config, kind, status, projectSegment) {
   }
 }
 function localMemoriesRoot(config) {
-  return (0, import_node_path15.join)(config.agentContextHome, "data", "viking", config.account, "user", uriSegment(config.user), "memories");
+  return (0, import_node_path16.join)(config.agentContextHome, "data", "viking", config.account, "user", uriSegment(config.user), "memories");
 }
 function localResourcesRoot(config) {
-  return (0, import_node_path15.join)(config.agentContextHome, "data", "viking", config.account, "resources");
+  return (0, import_node_path16.join)(config.agentContextHome, "data", "viking", config.account, "resources");
 }
 function localPathForMemoryUri(config, uri) {
   const prefix = `viking://user/${uriSegment(config.user)}/memories`;
@@ -15922,17 +16115,17 @@ function localPathForMemoryUri(config, uri) {
   if (segments.some((segment) => segment === "." || segment === "..")) {
     return void 0;
   }
-  return (0, import_node_path15.join)(localMemoriesRoot(config), ...segments);
+  return (0, import_node_path16.join)(localMemoriesRoot(config), ...segments);
 }
 function isMissingPathError(err) {
   return typeof err === "object" && err !== null && "code" in err && err.code === "ENOENT";
 }
 function localPathToMemoryUri(config, path2) {
-  const relativePath = (0, import_node_path15.relative)(localMemoriesRoot(config), path2);
-  if (!relativePath || relativePath.startsWith("..") || relativePath.split(import_node_path15.sep).includes("..")) {
+  const relativePath = (0, import_node_path16.relative)(localMemoriesRoot(config), path2);
+  if (!relativePath || relativePath.startsWith("..") || relativePath.split(import_node_path16.sep).includes("..")) {
     throw new Error(`Path is outside the memories tree: ${path2}`);
   }
-  return `viking://user/${uriSegment(config.user)}/memories/${relativePath.split(import_node_path15.sep).join("/")}`;
+  return `viking://user/${uriSegment(config.user)}/memories/${relativePath.split(import_node_path16.sep).join("/")}`;
 }
 async function ensurePersonalDirectoryChain2(config, ov, directoryUri) {
   const prefix = "viking://";
@@ -16172,6 +16365,9 @@ async function main() {
   program2.command("migrate-lifecycle").description("Move clear legacy handoff memories into lifecycle-aware archive paths").option("--apply", "Perform the migration; without this, prints a dry run").option("--dry-run", "Print migration actions without writing or removing memories").option("--limit <count>", "Maximum number of legacy handoffs to migrate").action(async (options) => {
     await runMigrateLifecycle(getRuntimeConfig(program2), options);
   });
+  program2.command("repair-semantic-queue").description("Patch the installed OpenViking to drain/avoid the semantic-queue poison loop (#2734)").option("--apply", "Apply the patch and restart the server; without this, prints a dry run").option("--dry-run", "Print what would change without patching or restarting").action(async (options) => {
+    await runRepairSemanticQueue(getRuntimeConfig(program2), options);
+  });
   program2.command("recall").description("Search shared OpenViking context").requiredOption("--query <query>", "Search query").option("--dry-run", "Print ov command without searching").option("--include-archived", "Include archived memories in recall results").option("-n, --node-limit <count>", "Maximum number of search results").option("--no-infer-scope", "Disable query-based scope inference").option("--project <name>", "Prioritize a project: add a scoped pass over its memories alongside the global search").option("--threshold <score>", "Minimum relevance score 0-1 (default 0.45); lower to broaden when recall is empty").option("--uri <uri>", "Restrict search to a viking:// URI").action(async (options) => {
     await runRecall(getRuntimeConfig(program2), options);
   });

package/docs/troubleshooting.md

@@ -94,6 +94,38 @@ threadnote doctor --dry-run
 If it still is not healthy, open that log. Certificate failures during the first embedding model download are covered
 above.
 
+## Semantic Queue Stuck / Memory Writes Hang
+
+Symptom: agents hang or `remember`/`handoff` get very slow, and `~/.openviking/logs/server.log` repeats:
+
+```
+RuntimeError: Failed to list memory directory viking://user/.../memories/.../<name>.md: Directory not found
+```
+
+A memory _file_ got enqueued for directory-level semantic processing; OpenViking's `_process_memory_directory` lists
+it, fails, and the message re-enqueues forever. The entry is AGFS-persisted, so it survives a server restart. Check the
+queue — a non-zero `Errors`/`Requeued` on the `Semantic` row is the signature:
+
+```bash
+ov observer queue
+```
+
+Fix it by patching the installed OpenViking and restarting the server:
+
+```bash
+threadnote repair-semantic-queue --apply
+```
+
+It skips non-directory/missing memory URIs (OpenViking PR #2735), keeps a `.threadnote-bak`, compile-checks the patched
+file before writing, and is a no-op once the installed OpenViking already includes the fix. This is a **temporary local
+patch** — `threadnote update` also offers it as a post-update step, and it is superseded automatically once Threadnote
+pins an OpenViking release containing the fix. To revert manually:
+
+```bash
+mv <printed-path>.threadnote-bak <printed-path>
+threadnote stop && threadnote start
+```
+
 ## Port Already In Use
 
 The default bind address is `127.0.0.1:1933`. This does not conflict with projects serving `localhost:80`,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "threadnote",
-  "version": "1.4.2",
+  "version": "1.4.3",
   "type": "commonjs",
   "main": "dist/threadnote.cjs",
   "description": "Shared local context and handoffs for development agents",