pi-quiver 5.2.4 → 5.3.0

package/dist/lib/unpdf-worker.js

@@ -0,0 +1,58 @@
+// lib/unpdf-worker.ts
+import { readFileSync } from "node:fs";
+import { getDocumentProxy } from "unpdf";
+var SEP = (n) => `--- end of page.page_number=${n} ---`;
+async function readStdin() {
+  const chunks = [];
+  for await (const c of process.stdin) chunks.push(c);
+  return Buffer.concat(chunks).toString("utf8");
+}
+async function main() {
+  const mode = process.argv[2];
+  if (mode !== "info" && mode !== "pdf-text") {
+    process.stderr.write("usage: unpdf-worker <info|pdf-text>\n");
+    return 1;
+  }
+  const o = JSON.parse(await readStdin() || "{}");
+  try {
+    const pdf = await getDocumentProxy(new Uint8Array(readFileSync(o.path)));
+    const pageCount = pdf.numPages;
+    if (mode === "info") {
+      const meta = (await pdf.getMetadata().catch(() => null))?.info;
+      const metadata = {};
+      for (const k of ["Title", "Author", "Subject", "Creator", "Producer"]) if (typeof meta?.[k] === "string" && meta[k]) metadata[k.toLowerCase()] = meta[k];
+      process.stdout.write(JSON.stringify({ pageCount, metadata, toc: [] }));
+      return 0;
+    }
+    const pages = o.pages ?? Array.from({ length: pageCount }, (_, i) => i + 1);
+    const bad = pages.filter((p) => p < 1 || p > pageCount);
+    if (bad.length) {
+      process.stdout.write(JSON.stringify({ error: `pages out of range: ${bad.join(", ")} (document has ${pageCount} pages)`, pageCount }));
+      return 3;
+    }
+    const out = [], emptyPages = [], failedPages = [];
+    for (const n of pages) {
+      let text = "";
+      try {
+        const page = await pdf.getPage(n);
+        const content = await page.getTextContent();
+        text = content.items.map((it) => "str" in it ? it.str + (it.hasEOL ? "\n" : " ") : "").join("").trim();
+      } catch (e) {
+        failedPages.push({ page: n, error: String(e).slice(0, 300) });
+      }
+      if (!text) emptyPages.push(n);
+      out.push(text, SEP(n));
+    }
+    if (failedPages.length === pages.length) throw new Error(`every selected page failed: ${failedPages[0].error}`);
+    process.stdout.write(JSON.stringify({ markdown: `${out.join("\n\n")}
+`, pages, pageCount, emptyPages, failedPages, notes: ["No images: unpdf backend"] }));
+    return 0;
+  } catch (e) {
+    process.stderr.write(`${e instanceof Error ? e.stack ?? e.message : String(e)}
+`);
+    return 1;
+  }
+}
+main().then((code) => {
+  process.exitCode = code;
+});

package/extensions/doc_to_md.ts

@@ -1,85 +1,75 @@
 /**
- * doc_to_md Extension
- *
- * Registers a `doc_to_md` tool that converts a local PDF, DOCX, or PPTX file
- * to Markdown. Primary engine: pymupdf4llm via ephemeral `uv run --with`
- * (warm-once per process, no repo venv). Fallback: unpdf pure-JS text
- * extraction (degraded, explicitly marked). DOCX/PPTX convert to PDF first
- * via headless soffice, then feed the PDF pipeline. Output over 32 KB or
- * 1000 lines is spilled to a temp .md file with a 60-line preview; smaller
- * content is returned inline.
+ * doc_to_md Extension - converts a local PDF/DOCX/PPTX/XLSX/XLS to a Markdown bundle on disk
+ * (<stem>.md + images/) and returns a bounded handle; `info: true` inspects without converting.
+ * Schema and settings shape derive from the core's option descriptors (single source of truth).
  */
-
 import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
-import { formatSize, keyHint } from "@earendil-works/pi-coding-agent";
+import { keyHint } from "@earendil-works/pi-coding-agent";
 import { Text } from "@earendil-works/pi-tui";
-import { Type } from "@sinclair/typebox";
-import { convertDocument, parseConfig, type DocToMdDetails } from "../lib/doc-to-md-core.ts";
+import { Type, type TObject, type TSchema } from "@sinclair/typebox";
+import { resolveConfig } from "../lib/extension-config.ts";
+import {
+	DOC_TO_MD_OPTIONS, type DocToMdDetails, type OptionDescriptor, type PerCallInput, type Tunables,
+	coerceDocToMdSettings, convertDocument, inspectDocument, resolveOptions,
+} from "../lib/doc-to-md-core.ts";
+import { resolve } from "node:path";
+
+function schemaFor(d: OptionDescriptor): TSchema {
+	const desc = { description: d.help };
+	switch (d.type) {
+		case "bool": return Type.Boolean(desc);
+		case "int": return Type.Integer({ ...desc, minimum: 1 });
+		case "enum": return Type.Union(d.enumValues!.map((v) => Type.Literal(v)), desc);
+		default: return Type.String(desc);
+	}
+}
+
+function buildSchema(): TObject {
+	const props: Record<string, TSchema> = {};
+	for (const d of DOC_TO_MD_OPTIONS) props[d.key] = d.key === "path" ? schemaFor(d) : Type.Optional(schemaFor(d));
+	return Type.Object(props);
+}
 
 export default function docToMdExtension(pi: ExtensionAPI) {
 	pi.registerTool({
 		name: "doc_to_md",
-		label: "Convert doc to Markdown",
+		label: "Convert doc to Markdown bundle",
 		description:
-			"Convert a local PDF/DOCX/PPTX file to Markdown. High-fidelity conversion via pymupdf4llm, resolved per process: uv (pinned, isolated) when available, else a system Python >= 3.12 with pymupdf4llm already installed, else a one-time managed venv bootstrapped into the user cache dir; falls back to a degraded pure-JS text extractor (unpdf) when no capable Python exists or conversion fails. DOCX/PPTX require LibreOffice (soffice) for the office->PDF step. Output over 32KB or 1000 lines is written to a temp .md file with a preview instead of inlined - grep it or read with offset/limit. A degraded result (marked in the output) means the fallback ran: tables and headings are NOT faithfully preserved, treat structure with suspicion. Input must be a local file path (use fetch first for URLs).",
-		promptSnippet: "Convert a local PDF/DOCX/PPTX to Markdown",
-		parameters: Type.Object({
-			path: Type.String({ description: "Local path to a .pdf, .docx, or .pptx file" }),
-		}),
+			"Convert a local PDF/DOCX/PPTX/XLSX/XLS to a Markdown bundle on disk and return a handle (Saved-To, Images-Dir, Page-Count, Outline, diagnostics) - the Markdown itself is never inlined; read the Saved-To file (offset/limit) for content. `info: true` returns page count, metadata and TOC (or the sheet inventory) without converting - use it to pick `pages`. `pages` selects inclusive 1-based pages (PDF/DOCX/PPTX); every page ends with `--- end of page.page_number=N ---`. Images are always extracted into images/ with relative links. Primary engine pymupdf4llm, fallback PyMuPDF text (degraded, marked), pure-JS unpdf only when no Python backend exists. Excel yields per-worksheet matrices with formulas, cached values, merged/hidden disclosure. DOCX/PPTX need LibreOffice (soffice). Use `outputDir` for a durable bundle; without it the bundle lands in a per-call temp dir. Input must be a local file path (use fetch first for URLs).",
+		promptSnippet: "Convert a local PDF/DOCX/PPTX/XLSX to a Markdown bundle (handle returned; read Saved-To)",
+		parameters: buildSchema(),
 
-		async execute(_toolCallId, params, signal) {
-			const cfg = parseConfig(process.env);
-			const { output, details } = await convertDocument(params.path, cfg, signal);
-			return { content: [{ type: "text" as const, text: output }], details };
+		async execute(_toolCallId, params, signal, _onUpdate, ctx) {
+			const settings = resolveConfig<Partial<Tunables>>(ctx.cwd, "docToMd", {}, (raw) => coerceDocToMdSettings(raw, (m) => console.warn(m)), (m) => console.warn(m));
+			const p = params as unknown as PerCallInput;
+			const perCall: PerCallInput = { ...p, path: resolve(ctx.cwd, p.path), ...(p.outputDir ? { outputDir: resolve(ctx.cwd, p.outputDir) } : {}) };
+			const o = resolveOptions(perCall, settings, process.env);
+			const r = o.info ? await inspectDocument(o, signal) : await convertDocument(o, signal);
+			return { content: [{ type: "text" as const, text: r.output }], details: r.details };
 		},
 
-		renderCall(args, theme, _context) {
-			let text = theme.fg("toolTitle", theme.bold("doc_to_md "));
-			text += theme.fg("accent", args.path ?? "");
+		renderCall(args, theme) {
+			const p = args as unknown as PerCallInput;
+			let text = theme.fg("toolTitle", theme.bold(p.info ? "doc_to_md --info " : "doc_to_md "));
+			text += theme.fg("accent", p.path ?? "");
+			if (p.pages) text += theme.fg("dim", ` pages ${p.pages}`);
 			return new Text(text, 0, 0);
 		},
 
 		renderResult(result, { expanded, isPartial }, theme, context) {
-			if (isPartial) {
-				return new Text(theme.fg("warning", "Converting..."), 0, 0);
-			}
-
-			const details = result.details as DocToMdDetails | undefined;
+			if (isPartial) return new Text(theme.fg("warning", "Converting..."), 0, 0);
 			const content = result.content[0];
 			const fullText = content?.type === "text" ? content.text : "";
-
-			if (context.isError) {
-				const firstLine = fullText.split("\n")[0] || "doc_to_md failed";
-				return new Text(theme.fg("error", firstLine), 0, 0);
-			}
-
+			if (context.isError) return new Text(theme.fg("error", fullText.split("\n")[0] || "doc_to_md failed"), 0, 0);
+			const d = result.details as Partial<DocToMdDetails> | undefined;
 			const sep = theme.fg("dim", " · ");
-			const parts: string[] = [];
-			parts.push(theme.fg("muted", details?.inputType ?? "?"));
-			parts.push(
-				details?.degraded
-					? theme.fg("warning", "unpdf (degraded)")
-					: theme.fg("muted", "pymupdf4llm"),
-			);
-			parts.push(theme.fg("dim", formatSize(details?.bytes ?? 0)));
-			if (details?.spilled) parts.push(theme.fg("warning", "→ file"));
-
+			const parts = [theme.fg("muted", d?.type ?? "?")];
+			if (d?.engine) parts.push(d.degraded ? theme.fg("warning", `${d.engine} (degraded)`) : theme.fg("muted", d.engine));
+			if (d?.pages !== undefined) parts.push(theme.fg("dim", d.pages ? `${d.pages.length} pages` : "all pages"));
+			if (d?.imageCount !== undefined) parts.push(theme.fg("dim", `${d.imageCount} images`));
 			let text = parts.join(sep);
-
-			if (!expanded) {
-				const lineCount = details?.lines ?? (fullText ? fullText.split("\n").length : 0);
-				if (lineCount > 0) {
-					text += sep + theme.fg("dim", `${lineCount} lines`);
-				}
-				text += " " + theme.fg("dim", `(${keyHint("app.tools.expand", "to expand")})`);
-				return new Text(text, 0, 0);
-			}
-
-			if (fullText) {
-				for (const line of fullText.split("\n")) {
-					text += `\n${theme.fg("toolOutput", line)}`;
-				}
-			}
+			if (!expanded) return new Text(`${text} ${theme.fg("dim", `(${keyHint("app.tools.expand", "to expand")})`)}`, 0, 0);
+			for (const line of fullText.split("\n")) text += `\n${theme.fg("toolOutput", line)}`;
 			return new Text(text, 0, 0);
 		},
 	});

package/lib/doc-to-md-bundle.ts

@@ -0,0 +1,139 @@
+/**
+ * Bundle protocol: a call owns `<stem>` for its whole duration via `<stem>.md.lock`;
+ * children stage images under `images/.stage-<lockId>/p<N>/` and mark pages `.done`;
+ * Node publishes done pages to `images/<stem>-p<N>-<n>.<ext>`, records every file it
+ * wrote in a manifest, and commits `<stem>.md` atomically (tmp + rename).
+ */
+import fs, { closeSync, existsSync, mkdirSync, openSync, readdirSync, readFileSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { extname, join, resolve } from "node:path";
+import { randomBytes } from "node:crypto";
+
+export interface Bundle {
+	root: string; stem: string; mdPath: string; lockPath: string; imagesDir: string; stagingDir: string; lockId: string;
+	manifest: Set<string>;
+	sourceMap: Map<string, string>;
+}
+
+export function ownedPattern(stem: string): RegExp {
+	const esc = stem.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+	return new RegExp(`^${esc}-(p|s)\\d+-\\d+\\.[a-z0-9]+$`);
+}
+
+const IMG_LINK_RE = /!\[[^\]]*\]\(\s*(?:<([^>]*)>|([^)]*?))\s*\)|<img\b[^>]*\bsrc\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>"']+))/gi;
+
+function imageTarget(m: RegExpMatchArray): string {
+	const target = (m[1] ?? m[2] ?? m[3] ?? m[4] ?? m[5] ?? "").trim();
+	const destination = target.replace(/^<([\s\S]*)>$/, "$1").trim();
+	return m[2] === undefined ? destination : destination.replace(/\s+(?:"[^"]*"|'[^']*'|\([^)]*\))$/, "");
+}
+
+function linkedFiles(md: string): string[] {
+	const out: string[] = [];
+	for (const m of md.matchAll(IMG_LINK_RE)) {
+		const target = imageTarget(m);
+		if (!target.startsWith("images/")) continue;
+		const file = target.slice("images/".length);
+		if (file && file !== "." && file !== ".." && /^[^/\\]+$/.test(file)) out.push(file);
+	}
+	return out;
+}
+
+export function openBundle(root: string, stem: string, overwrite: boolean): Bundle {
+	mkdirSync(root, { recursive: true });
+	const mdPath = join(root, `${stem}.md`);
+	const lockPath = `${mdPath}.lock`;
+	let fd: number;
+	try { fd = openSync(lockPath, "wx"); }
+	catch (e) {
+		if ((e as NodeJS.ErrnoException).code === "EEXIST") throw new Error(`Another conversion owns ${mdPath} (lock: ${lockPath}); if no conversion is running, delete the lock`);
+		throw new Error(`Output directory not writable: ${root} (${(e as Error).message})`);
+	}
+	closeSync(fd);
+	const imagesDir = join(root, "images");
+	try {
+		if (existsSync(mdPath)) {
+			if (!overwrite) throw new Error(`Output exists: ${mdPath} (pass overwrite)`);
+			const owned = ownedPattern(stem);
+			const linked = new Set(linkedFiles(readFileSync(mdPath, "utf8")));
+			unlinkSync(mdPath);
+			if (existsSync(imagesDir)) for (const f of readdirSync(imagesDir)) if (owned.test(f) || linked.has(f)) rmSync(join(imagesDir, f), { force: true });
+		}
+		const lockId = randomBytes(6).toString("hex");
+		const stagingDir = join(imagesDir, `.stage-${lockId}`);
+		mkdirSync(stagingDir, { recursive: true });
+		return { root, stem, mdPath, lockPath, imagesDir, stagingDir, lockId, manifest: new Set(), sourceMap: new Map() };
+	} catch (e) { rmSync(lockPath, { force: true }); throw e; }
+}
+
+/** Publish every `p<N>/` staging dir carrying `.done`; discard partial ones. Returns page -> published filenames. */
+export function publishStaged(b: Bundle): Map<number, string[]> {
+	const out = new Map<number, string[]>();
+	if (!existsSync(b.stagingDir)) return out;
+	for (const dir of readdirSync(b.stagingDir).sort()) {
+		const m = dir.match(/^p(\d+)$/);
+		if (!m) continue;
+		const pageDir = join(b.stagingDir, dir);
+		if (!existsSync(join(pageDir, ".done"))) { rmSync(pageDir, { recursive: true, force: true }); continue; }
+		const page = Number(m[1]);
+		const files = readdirSync(pageDir).filter((f) => f !== ".done" && statSync(join(pageDir, f)).isFile()).sort();
+		const names: string[] = [];
+		files.forEach((f, i) => {
+			const name = `${b.stem}-p${page}-${i + 1}${extname(f).toLowerCase()}`;
+			renameSync(join(pageDir, f), join(b.imagesDir, name));
+			b.manifest.add(name);
+			b.sourceMap.set(`${dir}/${f}`, `images/${name}`);
+			names.push(name);
+		});
+		rmSync(pageDir, { recursive: true, force: true });
+		out.set(page, names);
+	}
+	return out;
+}
+
+/** Excel: files staged flat in stagingDir as `s<idx>-<n>.<ext>` -> `images/<stem>-s<idx>-<n>.<ext>`. */
+export function publishSheetImages(b: Bundle): void {
+	for (const f of readdirSync(b.stagingDir).sort()) {
+		if (!/^s\d+-\d+\.[a-z0-9]+$/i.test(f)) continue;
+		const name = `${b.stem}-${f.toLowerCase()}`;
+		renameSync(join(b.stagingDir, f), join(b.imagesDir, name));
+		b.manifest.add(name);
+		b.sourceMap.set(f, `images/${name}`);
+	}
+}
+
+export function rewriteImageLinks(md: string, sourceMap: Map<string, string>): string {
+	return md.replace(IMG_LINK_RE, (whole, mdAngle, mdPlain, htmlDouble, htmlSingle, htmlUnquoted) => {
+		const target = imageTarget([whole, mdAngle, mdPlain, htmlDouble, htmlSingle, htmlUnquoted] as unknown as RegExpMatchArray);
+		const dest = sourceMap.get(target) ?? sourceMap.get(target.replace(/^\.\//, ""));
+		return dest ? whole.replace(mdAngle === undefined ? target : `<${mdAngle}>`, dest) : whole;
+	});
+}
+
+export function validateImageLinks(md: string, manifest: Set<string>): void {
+	for (const m of md.matchAll(IMG_LINK_RE)) {
+		const target = imageTarget(m);
+		if (!target.startsWith("images/") || !manifest.has(target.slice("images/".length))) throw new Error(`unexpected image reference in output: ${target}`);
+	}
+}
+
+export function commitBundle(b: Bundle, markdown: string): void {
+	const tmp = `${b.mdPath}.tmp`;
+	writeFileSync(tmp, markdown, "utf8");
+	renameSync(tmp, b.mdPath);
+	try { fs.rmSync(b.stagingDir, { recursive: true, force: true }); } catch { /* Markdown is published; cleanup is best-effort. */ }
+	try { fs.rmSync(b.lockPath, { force: true }); } catch { /* Markdown is published; cleanup is best-effort. */ }
+}
+
+export function abortBundle(b: Bundle): void {
+	for (const f of b.manifest) rmSync(join(b.imagesDir, f), { force: true });
+	rmSync(`${b.mdPath}.tmp`, { force: true });
+	rmSync(b.stagingDir, { recursive: true, force: true });
+	rmSync(b.lockPath, { force: true });
+}
+
+export function tempBundleRoot(): string {
+	const dir = join(tmpdir(), `pi-quiver-doc-to-md-${randomBytes(4).toString("hex")}`);
+	mkdirSync(dir, { recursive: true });
+	return resolve(dir);
+}