pi-quiver 5.2.4 → 5.3.0

package/lib/doc-to-md-handle.ts

@@ -0,0 +1,108 @@
+import type { InputType } from "./doc-to-md-options.ts";
+
+export type Tier = "primary" | "fallback" | "unpdf" | "excel";
+export type Engine = "pymupdf4llm" | "pymupdf-text" | "unpdf" | "openpyxl" | "xlrd";
+export type BackendKind = "uv" | "python" | "venv" | "none";
+
+export interface OutlineEntry { line: number; level: number; title: string; }
+export interface TocEntry { level: number; title: string; page: number; }
+export interface SheetInfo { index: number; name: string; hidden: boolean; rows: number | null; cols: number | null; hiddenRows: number; hiddenCols: number; }
+
+export interface HandleData {
+	savedTo: string; imagesDir: string | null; type: InputType; engine: Engine; tier: Tier;
+	pageCount: number | null; pages: number[] | null; imageCount: number; bytes: number; lines: number;
+	degraded: string | null; fallbackReason: string | null; failedPages: number[]; emptyPages: number[];
+	notes: string[]; outline: OutlineEntry[]; outlineTotal: number;
+}
+
+export interface InfoData {
+	type: InputType; backend: BackendKind; pageCount: number | null; metadata: Record<string, string>;
+	toc: TocEntry[]; tocTotal: number; sheets: SheetInfo[] | null; sheetsTotal: number;
+}
+
+const TITLE_MAX = 80;
+const NOTE_MAX_LINES = 5;
+const NOTE_MAX_CHARS = 200;
+const META_MAX_CHARS = 120;
+
+// Deliberate local copy of the host package's formatSize - this module must stay pi-free.
+export function formatSize(bytes: number): string {
+	if (bytes < 1024) return `${bytes}B`;
+	if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)}KB`;
+	return `${(bytes / (1024 * 1024)).toFixed(1)}MB`;
+}
+
+const trunc = (s: string, max: number) => (s.length > max ? `${s.slice(0, max - 3)}...` : s);
+
+export function compactRanges(nums: number[], maxEntries = 20): string {
+	const sorted = [...new Set(nums)].sort((a, b) => a - b);
+	const parts: string[] = [];
+	for (let i = 0; i < sorted.length;) {
+		let j = i;
+		while (j + 1 < sorted.length && sorted[j + 1] === sorted[j] + 1) j++;
+		parts.push(j > i ? `${sorted[i]}-${sorted[j]}` : `${sorted[i]}`);
+		i = j + 1;
+	}
+	if (parts.length <= maxEntries) return parts.join(", ");
+	return `${parts.slice(0, maxEntries).join(", ")} (+${parts.length - maxEntries} more)`;
+}
+
+export function scanOutline(md: string, max: number): { entries: OutlineEntry[]; total: number } {
+	const entries: OutlineEntry[] = [];
+	let total = 0, inFence = false;
+	md.split("\n").forEach((raw, i) => {
+		const line = raw.replace(/\r$/, "");
+		if (/^\s*(```|~~~)/.test(line)) { inFence = !inFence; return; }
+		if (inFence) return;
+		const m = line.match(/^(#{1,6}) (.*)$/);
+		if (!m) return;
+		total++;
+		if (entries.length < max) entries.push({ line: i + 1, level: m[1].length, title: trunc(m[2].trim(), TITLE_MAX) });
+	});
+	return { entries, total };
+}
+
+function outlineLines(entries: OutlineEntry[], total: number): string[] {
+	if (entries.length === 0) return [];
+	const width = Math.max(5, Math.max(...entries.map((e) => `L${e.line}`.length)) + 2);
+	const out = ["Outline:", ...entries.map((e) => `  ${`L${e.line}`.padEnd(width)}${"#".repeat(e.level)} ${trunc(e.title, TITLE_MAX)}`)];
+	if (total > entries.length) out.push(`  (+${total - entries.length} more)`);
+	return out;
+}
+
+export function formatHandle(h: HandleData): string {
+	const lines = [`Saved-To: ${h.savedTo}`];
+	if (h.imagesDir && h.imageCount > 0) lines.push(`Images-Dir: ${h.imagesDir}`);
+	lines.push(`Type: ${h.type}   Engine: ${h.engine}   Tier: ${h.tier}`);
+	lines.push(`Page-Count: ${h.pageCount ?? "?"}   Pages: ${h.pages ? compactRanges(h.pages) : "all"}   Images: ${h.imageCount}   Size: ${formatSize(h.bytes)} / ${h.lines} lines`);
+	if (h.degraded) lines.push(`Degraded: ${h.degraded}`);
+	if (h.fallbackReason) lines.push(`Fallback-Reason: ${h.fallbackReason}`);
+	const fe: string[] = [];
+	if (h.failedPages.length) fe.push(`Failed-Pages: ${compactRanges(h.failedPages)}`);
+	if (h.emptyPages.length) fe.push(`Empty-Pages: ${compactRanges(h.emptyPages)}`);
+	if (fe.length) lines.push(fe.join("    "));
+	h.notes.slice(0, NOTE_MAX_LINES).forEach((n, i) => lines.push(`${i === 0 ? "Notes: " : "       "}${trunc(n, NOTE_MAX_CHARS)}`));
+	lines.push(...outlineLines(h.outline, h.outlineTotal));
+	return lines.join("\n");
+}
+
+export function formatInfoHandle(i: InfoData, max: number): string {
+	if (i.sheets) {
+		const lines = [`Type: ${i.type}   Sheets: ${i.sheetsTotal}`];
+		for (const s of i.sheets.slice(0, max)) {
+			const dims = `rows=${s.rows ?? "?"} cols=${s.cols ?? "?"}`;
+			const hidden = s.hiddenRows || s.hiddenCols ? ` hiddenRows=${s.hiddenRows} hiddenCols=${s.hiddenCols}` : "";
+			lines.push(`  ${trunc(s.name, TITLE_MAX)}  ${s.hidden ? "hidden " : ""}${dims}${hidden}`);
+		}
+		if (i.sheetsTotal > max) lines.push(`  (+${i.sheetsTotal - max} more)`);
+		return lines.join("\n");
+	}
+	const lines = [`Type: ${i.type}   Page-Count: ${i.pageCount ?? "?"}   Backend: ${i.backend}`];
+	const meta = Object.entries(i.metadata).filter(([, v]) => v).map(([k, v]) => `${k[0].toUpperCase()}${k.slice(1)}: ${trunc(v, META_MAX_CHARS)}`);
+	if (meta.length) lines.push(meta.join("   "));
+	if (i.toc.length) {
+		lines.push("TOC:", ...i.toc.slice(0, max).map((t) => `  L${t.level} ${trunc(t.title, TITLE_MAX)} (p${t.page})`));
+		if (i.tocTotal > max) lines.push(`  (+${i.tocTotal - max} more)`);
+	}
+	return lines.join("\n");
+}

package/lib/doc-to-md-options.ts

@@ -0,0 +1,186 @@
+/**
+ * doc_to_md option descriptors - the single source of truth for the pi tool schema,
+ * the CLI flags/--help, and the `quiver.docToMd` settings shape. pi-free.
+ */
+import { extname } from "node:path";
+
+export type InputType = "pdf" | "docx" | "pptx" | "xlsx" | "xls";
+export type ImageFormat = "png" | "jpg";
+
+export interface Tunables {
+	primaryTimeoutMs: number;
+	fallbackTimeoutMs: number;
+	sofficeTimeoutMs: number;
+	excelTimeoutMs: number;
+	warmTimeoutMs: number;
+	pymupdfVersion: string;
+	imageDpi: number;
+	imageFormat: ImageFormat;
+	maxCellsPerSheet: number;
+	maxOutputBytes: number;
+	outlineMaxEntries: number;
+}
+
+export interface DocToMdOptions extends Tunables {
+	path: string;
+	info: boolean;
+	pages: number[] | null;
+	outputDir: string | null;
+	overwrite: boolean;
+}
+
+/** What adapters pass in: intents as raw strings/booleans, tunables optional. */
+export interface PerCallInput extends Partial<Tunables> {
+	path: string;
+	info?: boolean;
+	pages?: string | null;
+	outputDir?: string | null;
+	overwrite?: boolean;
+}
+
+export type DescriptorType = "string" | "int" | "bool" | "pages" | "enum" | "version";
+
+export interface OptionDescriptor {
+	key: keyof DocToMdOptions;
+	flag: string | null;
+	type: DescriptorType;
+	default: string | number | boolean | null;
+	settable: boolean;
+	env?: string;
+	enumValues?: readonly string[];
+	help: string;
+}
+
+export class UsageError extends Error {}
+
+export const MIN_PYMUPDF4LLM = "1.27.0";
+const VERSION_RE = /^\d+(\.\d+)*$/;
+
+export const DOC_TO_MD_OPTIONS: readonly OptionDescriptor[] = [
+	{ key: "path", flag: null, type: "string", default: null, settable: false, help: "Local .pdf .docx .pptx .xlsx .xls file" },
+	{ key: "info", flag: "--info", type: "bool", default: false, settable: false, help: "Inspect only (page count, metadata, TOC or sheet inventory); no bundle" },
+	{ key: "pages", flag: "--pages", type: "pages", default: null, settable: false, help: "Inclusive 1-based pages, e.g. \"12-15\" or \"3,7,10-12\" (PDF/DOCX/PPTX only); default all" },
+	{ key: "outputDir", flag: "--output-dir", type: "string", default: null, settable: false, help: "Bundle root for <stem>.md + images/; default a per-call temp dir" },
+	{ key: "overwrite", flag: "--overwrite", type: "bool", default: false, settable: false, help: "Replace an existing completed <stem>.md bundle" },
+	{ key: "primaryTimeoutMs", flag: "--primary-timeout", type: "int", default: 60000, settable: true, env: "PI_DOC_TO_MD_CONVERT_TIMEOUT_MS", help: "pymupdf4llm tier; also the unpdf tier" },
+	{ key: "fallbackTimeoutMs", flag: "--fallback-timeout", type: "int", default: 30000, settable: true, help: "PyMuPDF get_text tier; also info on PDF" },
+	{ key: "sofficeTimeoutMs", flag: "--soffice-timeout", type: "int", default: 120000, settable: true, env: "PI_DOC_TO_MD_SOFFICE_TIMEOUT_MS", help: "DOCX/PPTX -> PDF via LibreOffice" },
+	{ key: "excelTimeoutMs", flag: "--excel-timeout", type: "int", default: 60000, settable: true, help: "Excel child (both openpyxl loads); also info on Excel" },
+	{ key: "warmTimeoutMs", flag: "--warm-timeout", type: "int", default: 120000, settable: true, env: "PI_DOC_TO_MD_WARM_TIMEOUT_MS", help: "Absolute backend discovery/bootstrap deadline (first call per process)" },
+	{ key: "pymupdfVersion", flag: "--pymupdf-version", type: "version", default: "1.27.2.3", settable: true, env: "PI_DOC_TO_MD_PYMUPDF_VERSION", help: "pymupdf4llm pin (>= 1.27.0)" },
+	{ key: "imageDpi", flag: "--image-dpi", type: "int", default: 150, settable: true, help: "Render DPI for page images" },
+	{ key: "imageFormat", flag: "--image-format", type: "enum", default: "png", settable: true, enumValues: ["png", "jpg"], help: "Rendered image format (embedded images keep their native extension)" },
+	{ key: "maxCellsPerSheet", flag: "--max-cells-per-sheet", type: "int", default: 50000, settable: true, help: "rows x cols budget per worksheet" },
+	{ key: "maxOutputBytes", flag: "--max-output-bytes", type: "int", default: 20000000, settable: true, help: "Child stdout cap in bytes" },
+	{ key: "outlineMaxEntries", flag: "--outline-max-entries", type: "int", default: 40, settable: true, help: "Heading outline / TOC / sheet inventory cap in the handle" },
+];
+
+export const TUNABLE_DEFAULTS: Tunables = Object.fromEntries(
+	DOC_TO_MD_OPTIONS.filter((d) => d.settable).map((d) => [d.key, d.default]),
+) as unknown as Tunables;
+
+export function versionAtLeast(v: string, min: string): boolean {
+	const a = v.split(".").map(Number), b = min.split(".").map(Number);
+	for (let i = 0; i < Math.max(a.length, b.length); i++) {
+		const x = a[i] ?? 0, y = b[i] ?? 0;
+		if (x !== y) return x > y;
+	}
+	return true;
+}
+
+function coerceValue(d: OptionDescriptor, raw: unknown, fromString = false): { ok: true; value: unknown } | { ok: false; reason: string } {
+	switch (d.type) {
+		case "int": {
+			const n = fromString && typeof raw === "string" ? Number(raw) : raw;
+			return typeof n === "number" && Number.isInteger(n) && n > 0 ? { ok: true, value: n } : { ok: false, reason: "must be a positive integer" };
+		}
+		case "bool": return typeof raw === "boolean" ? { ok: true, value: raw } : { ok: false, reason: "must be true or false" };
+		case "enum": return typeof raw === "string" && d.enumValues!.includes(raw) ? { ok: true, value: raw } : { ok: false, reason: `must be one of ${d.enumValues!.join(", ")}` };
+		case "version":
+			if (typeof raw !== "string" || !VERSION_RE.test(raw)) return { ok: false, reason: "must be digits and dots" };
+			if (!versionAtLeast(raw, MIN_PYMUPDF4LLM)) return { ok: false, reason: `must be >= ${MIN_PYMUPDF4LLM}` };
+			return { ok: true, value: raw };
+		case "string": return typeof raw === "string" && raw.length > 0 ? { ok: true, value: raw } : { ok: false, reason: "must be a non-empty string" };
+		case "pages": return typeof raw === "string" ? { ok: true, value: parsePages(raw) } : { ok: false, reason: "must be a string" };
+	}
+}
+
+/** Single validation boundary for `quiver.docToMd`: unknown/ill-typed/intent keys are dropped with one warning each. */
+export function coerceDocToMdSettings(raw: unknown, warn: (message: string) => void = console.warn): Partial<Tunables> | undefined {
+	if (raw === null || typeof raw !== "object" || Array.isArray(raw)) return undefined;
+	const out: Record<string, unknown> = {};
+	for (const [k, v] of Object.entries(raw as Record<string, unknown>)) {
+		const d = DOC_TO_MD_OPTIONS.find((o) => o.key === k);
+		if (!d || !d.settable) { warn(`pi-quiver: quiver.docToMd.${k} is not a tunable setting; ignored.`); continue; }
+		const c = coerceValue(d, v);
+		if (!c.ok) { warn(`pi-quiver: quiver.docToMd.${k} ${c.reason}; ignored.`); continue; }
+		out[k] = c.value;
+	}
+	return out as Partial<Tunables>;
+}
+
+export function parsePages(spec: string): number[] {
+	const out = new Set<number>();
+	const parts = spec.split(",").map((s) => s.trim());
+	if (parts.length === 0 || parts.some((p) => p === "")) throw new UsageError(`invalid --pages "${spec}": expected e.g. "12-15" or "3,7,10-12"`);
+	for (const p of parts) {
+		const m = p.match(/^(\d+)(?:-(\d+))?$/);
+		if (!m) throw new UsageError(`invalid --pages "${spec}": bad token "${p}"`);
+		const a = Number(m[1]), b = m[2] === undefined ? a : Number(m[2]);
+		if (a < 1 || b < a) throw new UsageError(`invalid --pages "${spec}": pages are 1-based and ranges ascend`);
+		for (let i = a; i <= b; i++) out.add(i);
+	}
+	return [...out].sort((x, y) => x - y);
+}
+
+export function sanitizeStem(base: string): string {
+	const s = base.replace(/[^A-Za-z0-9._-]+/g, "_");
+	return s.length ? s : "document";
+}
+
+const SUPPORTED: Record<string, InputType> = { ".pdf": "pdf", ".docx": "docx", ".pptx": "pptx", ".xlsx": "xlsx", ".xls": "xls" };
+
+export function classifyInput(filePath: string): InputType {
+	const t = SUPPORTED[extname(filePath).toLowerCase()];
+	if (!t) throw new Error(`Unsupported file type "${extname(filePath) || "(none)"}"; supported: .pdf, .docx, .pptx, .xlsx, .xls`);
+	return t;
+}
+
+/** per-call > settings > deprecated env > default. Throws UsageError for bad per-call values or info + bundle option. */
+export function resolveOptions(perCall: PerCallInput, settings: Partial<Tunables>, env: NodeJS.ProcessEnv): DocToMdOptions {
+	const out: Record<string, unknown> = { path: perCall.path };
+	for (const d of DOC_TO_MD_OPTIONS) {
+		if (d.key === "path") continue;
+		let value: unknown = d.default;
+		if (d.env && env[d.env] !== undefined) {
+			const c = coerceValue(d, env[d.env], true);
+			if (!c.ok) throw new UsageError(`${d.env} ${c.reason} (got "${env[d.env]}")`);
+			value = c.value;
+		}
+		if (d.settable && (settings as Record<string, unknown>)[d.key] !== undefined) value = (settings as Record<string, unknown>)[d.key];
+		const pc = (perCall as unknown as Record<string, unknown>)[d.key];
+		if (pc !== undefined && pc !== null) {
+			const c = coerceValue(d, pc);
+			if (!c.ok) throw new UsageError(`${d.flag} ${c.reason}`);
+			value = c.value;
+		}
+		out[d.key] = value;
+	}
+	const o = out as unknown as DocToMdOptions;
+	if (o.info && (o.pages !== null || o.outputDir !== null || o.overwrite)) {
+		throw new UsageError("--info cannot be combined with --pages, --output-dir or --overwrite");
+	}
+	return o;
+}
+
+export function renderHelp(): string {
+	const row = (d: OptionDescriptor) => `  ${(d.flag ?? "<path>").padEnd(26)} ${d.help}${d.default !== null && d.key !== "info" && d.key !== "overwrite" ? ` (default ${d.default})` : ""}`;
+	return [
+		"Usage: pi-quiver doc-to-md [flags] <path>",
+		"", "Per-call:", ...DOC_TO_MD_OPTIONS.filter((d) => !d.settable).map(row),
+		"", "Tunables (also settable under quiver.docToMd in settings.json; per-call > settings > default):",
+		...DOC_TO_MD_OPTIONS.filter((d) => d.settable).map(row),
+		"", "Result: a handle (Saved-To, Images-Dir, Page-Count, Outline ...). Read the Saved-To file for the Markdown.",
+		"Exit codes: 0 success, 1 runtime error, 2 usage error.",
+	].join("\n");
+}

package/lib/unpdf-worker.ts

@@ -0,0 +1,54 @@
+/**
+ * unpdf child (Node). argv[2] = mode (info | pdf-text); options JSON on stdin; JSON result on stdout.
+ * Exit 0 ok, 1 failure, 3 user error. Runs as its own process so a PDF.js stall is killable.
+ * Only selected pages are requested (getPage(n)), so an unselected toxic page cannot burn the budget.
+ */
+import { readFileSync } from "node:fs";
+import { getDocumentProxy } from "unpdf";
+
+const SEP = (n: number) => `--- end of page.page_number=${n} ---`;
+
+async function readStdin(): Promise<string> {
+	const chunks: Buffer[] = [];
+	for await (const c of process.stdin) chunks.push(c as Buffer);
+	return Buffer.concat(chunks).toString("utf8");
+}
+
+async function main(): Promise<number> {
+	const mode = process.argv[2];
+	if (mode !== "info" && mode !== "pdf-text") { process.stderr.write("usage: unpdf-worker <info|pdf-text>\n"); return 1; }
+	const o = JSON.parse((await readStdin()) || "{}") as { path: string; pages?: number[] | null };
+	try {
+		const pdf = await getDocumentProxy(new Uint8Array(readFileSync(o.path)));
+		const pageCount = pdf.numPages;
+		if (mode === "info") {
+			const meta = (await pdf.getMetadata().catch(() => null))?.info as Record<string, unknown> | undefined;
+			const metadata: Record<string, string> = {};
+			for (const k of ["Title", "Author", "Subject", "Creator", "Producer"]) if (typeof meta?.[k] === "string" && meta[k]) metadata[k.toLowerCase()] = meta[k] as string;
+			process.stdout.write(JSON.stringify({ pageCount, metadata, toc: [] }));
+			return 0;
+		}
+		const pages = o.pages ?? Array.from({ length: pageCount }, (_, i) => i + 1);
+		const bad = pages.filter((p) => p < 1 || p > pageCount);
+		if (bad.length) { process.stdout.write(JSON.stringify({ error: `pages out of range: ${bad.join(", ")} (document has ${pageCount} pages)`, pageCount })); return 3; }
+		const out: string[] = [], emptyPages: number[] = [], failedPages: { page: number; error: string }[] = [];
+		for (const n of pages) {
+			let text = "";
+			try {
+				const page = await pdf.getPage(n);
+				const content = await page.getTextContent();
+				text = content.items.map((it) => ("str" in it ? it.str + (it.hasEOL ? "\n" : " ") : "")).join("").trim();
+			} catch (e) { failedPages.push({ page: n, error: String(e).slice(0, 300) }); }
+			if (!text) emptyPages.push(n);
+			out.push(text, SEP(n));
+		}
+		if (failedPages.length === pages.length) throw new Error(`every selected page failed: ${failedPages[0].error}`);
+		process.stdout.write(JSON.stringify({ markdown: `${out.join("\n\n")}\n`, pages, pageCount, emptyPages, failedPages, notes: ["No images: unpdf backend"] }));
+		return 0;
+	} catch (e) {
+		process.stderr.write(`${e instanceof Error ? e.stack ?? e.message : String(e)}\n`);
+		return 1;
+	}
+}
+
+main().then((code) => { process.exitCode = code; });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-quiver",
-  "version": "5.2.4",
+  "version": "5.3.0",
   "description": "Personal pack of Pi coding-agent extensions: context-safe fetch, doc_to_md PDF/DOCX/PPTX-to-Markdown conversion, session naming, a themed ASCII startup header, Opus 4.8 fast mode, and a provider-stall watchdog.",
   "author": "Jacek Juraszek",
   "license": "MIT",
@@ -22,6 +22,8 @@
     "pdf",
     "docx",
     "pptx",
+    "xlsx",
+    "xls",
     "cli"
   ],
   "engines": {
@@ -34,7 +36,7 @@
     "extensions",
     "lib",
     "dist",
-    "scripts/pdf_to_md.py",
+    "scripts/doc_to_md.py",
     "types/**/*.d.ts",
     "README.md",
     "CHANGELOG.md"
@@ -44,7 +46,7 @@
     "test": "node --test \"test/*.test.ts\"",
     "typecheck": "npx -y tsc --noEmit",
     "test:all": "npm run check:agents-core && npm run test && npm run typecheck",
-    "build": "esbuild bin/pi-quiver.ts --bundle --packages=external --format=esm --platform=node --outfile=dist/bin/pi-quiver.js",
+    "build": "esbuild bin/pi-quiver.ts lib/unpdf-worker.ts --bundle --packages=external --format=esm --platform=node --outdir=dist --outbase=.",
     "prepack": "npm run build"
   },
   "pi": {
@@ -82,7 +84,7 @@
     "jsdom": "^26.0.0",
     "turndown": "^7.2.0",
     "turndown-plugin-gfm": "^1.0.2",
-    "unpdf": "^1.6.0"
+    "unpdf": "^1.8.1"
   },
   "devDependencies": {
     "@earendil-works/pi-ai": "^0.84.2",