pi-quiver 5.2.4 → 5.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/README.md +36 -10
- package/dist/bin/pi-quiver.js +702 -259
- package/dist/lib/unpdf-worker.js +58 -0
- package/extensions/doc_to_md.ts +52 -62
- package/lib/doc-to-md-bundle.ts +139 -0
- package/lib/doc-to-md-core.ts +261 -301
- package/lib/doc-to-md-handle.ts +108 -0
- package/lib/doc-to-md-options.ts +186 -0
- package/lib/unpdf-worker.ts +54 -0
- package/package.json +6 -4
- package/scripts/doc_to_md.py +369 -0
- package/scripts/pdf_to_md.py +0 -30
package/dist/bin/pi-quiver.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
// bin/pi-quiver.ts
|
|
4
|
-
import { realpathSync } from "node:fs";
|
|
4
|
+
import { existsSync as existsSync3, readFileSync as readFileSync2, realpathSync } from "node:fs";
|
|
5
|
+
import { homedir as homedir2 } from "node:os";
|
|
6
|
+
import { join as join4, resolve as resolve3 } from "node:path";
|
|
5
7
|
import { fileURLToPath as fileURLToPath2 } from "node:url";
|
|
6
|
-
import { resolve as resolve2 } from "node:path";
|
|
7
8
|
|
|
8
9
|
// lib/fetch-core.ts
|
|
9
10
|
import { mkdirSync, writeFileSync, createWriteStream } from "node:fs";
|
|
@@ -318,8 +319,8 @@ function binaryExtension(contentType) {
|
|
|
318
319
|
return sub || "bin";
|
|
319
320
|
}
|
|
320
321
|
function writeChunk(stream, b) {
|
|
321
|
-
return new Promise((
|
|
322
|
-
stream.write(b, (err) => err ? reject(err) :
|
|
322
|
+
return new Promise((resolve4, reject) => {
|
|
323
|
+
stream.write(b, (err) => err ? reject(err) : resolve4());
|
|
323
324
|
});
|
|
324
325
|
}
|
|
325
326
|
async function pumpToFile(stream, reader, prefix, exhausted) {
|
|
@@ -343,7 +344,7 @@ async function pumpToFile(stream, reader, prefix, exhausted) {
|
|
|
343
344
|
await writeChunk(stream, chunk);
|
|
344
345
|
bytes += chunk.length;
|
|
345
346
|
}
|
|
346
|
-
await new Promise((
|
|
347
|
+
await new Promise((resolve4, reject) => stream.end((err) => err ? reject(err) : resolve4()));
|
|
347
348
|
return { bytes, truncated };
|
|
348
349
|
}
|
|
349
350
|
async function collectBody(res, contentType, raw) {
|
|
@@ -516,93 +517,383 @@ async function fetchUrl(opts) {
|
|
|
516
517
|
}
|
|
517
518
|
|
|
518
519
|
// lib/doc-to-md-core.ts
|
|
519
|
-
import { existsSync
|
|
520
|
+
import { existsSync as existsSync2, mkdtempSync, renameSync as renameSync2, rmSync as rmSync2, statSync as statSync2 } from "node:fs";
|
|
520
521
|
import { spawn } from "node:child_process";
|
|
521
|
-
import { homedir, tmpdir as
|
|
522
|
-
import { basename, dirname, extname, join as
|
|
523
|
-
import { createHash as createHash2 } from "node:crypto";
|
|
522
|
+
import { homedir, tmpdir as tmpdir3 } from "node:os";
|
|
523
|
+
import { basename, dirname, extname as extname3, join as join3, resolve as resolve2 } from "node:path";
|
|
524
524
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
var
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
525
|
+
|
|
526
|
+
// lib/doc-to-md-bundle.ts
|
|
527
|
+
import fs, { closeSync, existsSync, mkdirSync as mkdirSync2, openSync, readdirSync, readFileSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync as writeFileSync2 } from "node:fs";
|
|
528
|
+
import { tmpdir as tmpdir2 } from "node:os";
|
|
529
|
+
import { extname, join as join2, resolve } from "node:path";
|
|
530
|
+
import { randomBytes } from "node:crypto";
|
|
531
|
+
function ownedPattern(stem) {
|
|
532
|
+
const esc = stem.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
533
|
+
return new RegExp(`^${esc}-(p|s)\\d+-\\d+\\.[a-z0-9]+$`);
|
|
534
|
+
}
|
|
535
|
+
var IMG_LINK_RE = /!\[[^\]]*\]\(\s*(?:<([^>]*)>|([^)]*?))\s*\)|<img\b[^>]*\bsrc\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>"']+))/gi;
|
|
536
|
+
function imageTarget(m) {
|
|
537
|
+
const target = (m[1] ?? m[2] ?? m[3] ?? m[4] ?? m[5] ?? "").trim();
|
|
538
|
+
const destination = target.replace(/^<([\s\S]*)>$/, "$1").trim();
|
|
539
|
+
return m[2] === void 0 ? destination : destination.replace(/\s+(?:"[^"]*"|'[^']*'|\([^)]*\))$/, "");
|
|
540
|
+
}
|
|
541
|
+
function linkedFiles(md) {
|
|
542
|
+
const out = [];
|
|
543
|
+
for (const m of md.matchAll(IMG_LINK_RE)) {
|
|
544
|
+
const target = imageTarget(m);
|
|
545
|
+
if (!target.startsWith("images/")) continue;
|
|
546
|
+
const file = target.slice("images/".length);
|
|
547
|
+
if (file && file !== "." && file !== ".." && /^[^/\\]+$/.test(file)) out.push(file);
|
|
543
548
|
}
|
|
544
|
-
|
|
545
|
-
const raw = env[key];
|
|
546
|
-
if (raw === void 0) return def;
|
|
547
|
-
const n = Number.parseInt(raw, 10);
|
|
548
|
-
if (!Number.isInteger(n) || n <= 0) throw new Error(`${key} must be a positive integer (got "${raw}")`);
|
|
549
|
-
return n;
|
|
550
|
-
};
|
|
551
|
-
return {
|
|
552
|
-
pymupdfVersion: version,
|
|
553
|
-
warmTimeoutMs: num("PI_DOC_TO_MD_WARM_TIMEOUT_MS", WARM_TIMEOUT_DEFAULT),
|
|
554
|
-
convertTimeoutMs: num("PI_DOC_TO_MD_CONVERT_TIMEOUT_MS", CONVERT_TIMEOUT_DEFAULT),
|
|
555
|
-
sofficeTimeoutMs: num("PI_DOC_TO_MD_SOFFICE_TIMEOUT_MS", SOFFICE_TIMEOUT_DEFAULT)
|
|
556
|
-
};
|
|
549
|
+
return out;
|
|
557
550
|
}
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
const
|
|
561
|
-
const
|
|
562
|
-
|
|
563
|
-
|
|
551
|
+
function openBundle(root, stem, overwrite) {
|
|
552
|
+
mkdirSync2(root, { recursive: true });
|
|
553
|
+
const mdPath = join2(root, `${stem}.md`);
|
|
554
|
+
const lockPath = `${mdPath}.lock`;
|
|
555
|
+
let fd;
|
|
556
|
+
try {
|
|
557
|
+
fd = openSync(lockPath, "wx");
|
|
558
|
+
} catch (e) {
|
|
559
|
+
if (e.code === "EEXIST") throw new Error(`Another conversion owns ${mdPath} (lock: ${lockPath}); if no conversion is running, delete the lock`);
|
|
560
|
+
throw new Error(`Output directory not writable: ${root} (${e.message})`);
|
|
561
|
+
}
|
|
562
|
+
closeSync(fd);
|
|
563
|
+
const imagesDir = join2(root, "images");
|
|
564
|
+
try {
|
|
565
|
+
if (existsSync(mdPath)) {
|
|
566
|
+
if (!overwrite) throw new Error(`Output exists: ${mdPath} (pass overwrite)`);
|
|
567
|
+
const owned = ownedPattern(stem);
|
|
568
|
+
const linked = new Set(linkedFiles(readFileSync(mdPath, "utf8")));
|
|
569
|
+
unlinkSync(mdPath);
|
|
570
|
+
if (existsSync(imagesDir)) {
|
|
571
|
+
for (const f of readdirSync(imagesDir)) if (owned.test(f) || linked.has(f)) rmSync(join2(imagesDir, f), { force: true });
|
|
572
|
+
}
|
|
573
|
+
}
|
|
574
|
+
const lockId = randomBytes(6).toString("hex");
|
|
575
|
+
const stagingDir = join2(imagesDir, `.stage-${lockId}`);
|
|
576
|
+
mkdirSync2(stagingDir, { recursive: true });
|
|
577
|
+
return { root, stem, mdPath, lockPath, imagesDir, stagingDir, lockId, manifest: /* @__PURE__ */ new Set(), sourceMap: /* @__PURE__ */ new Map() };
|
|
578
|
+
} catch (e) {
|
|
579
|
+
rmSync(lockPath, { force: true });
|
|
580
|
+
throw e;
|
|
581
|
+
}
|
|
564
582
|
}
|
|
565
|
-
function
|
|
566
|
-
const
|
|
567
|
-
|
|
568
|
-
const
|
|
569
|
-
|
|
583
|
+
function publishStaged(b) {
|
|
584
|
+
const out = /* @__PURE__ */ new Map();
|
|
585
|
+
if (!existsSync(b.stagingDir)) return out;
|
|
586
|
+
for (const dir of readdirSync(b.stagingDir).sort()) {
|
|
587
|
+
const m = dir.match(/^p(\d+)$/);
|
|
588
|
+
if (!m) continue;
|
|
589
|
+
const pageDir = join2(b.stagingDir, dir);
|
|
590
|
+
if (!existsSync(join2(pageDir, ".done"))) {
|
|
591
|
+
rmSync(pageDir, { recursive: true, force: true });
|
|
592
|
+
continue;
|
|
593
|
+
}
|
|
594
|
+
const page = Number(m[1]);
|
|
595
|
+
const files = readdirSync(pageDir).filter((f) => f !== ".done" && statSync(join2(pageDir, f)).isFile()).sort();
|
|
596
|
+
const names = [];
|
|
597
|
+
files.forEach((f, i) => {
|
|
598
|
+
const name = `${b.stem}-p${page}-${i + 1}${extname(f).toLowerCase()}`;
|
|
599
|
+
renameSync(join2(pageDir, f), join2(b.imagesDir, name));
|
|
600
|
+
b.manifest.add(name);
|
|
601
|
+
b.sourceMap.set(`${dir}/${f}`, `images/${name}`);
|
|
602
|
+
names.push(name);
|
|
603
|
+
});
|
|
604
|
+
rmSync(pageDir, { recursive: true, force: true });
|
|
605
|
+
out.set(page, names);
|
|
606
|
+
}
|
|
607
|
+
return out;
|
|
570
608
|
}
|
|
571
|
-
function
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
609
|
+
function publishSheetImages(b) {
|
|
610
|
+
for (const f of readdirSync(b.stagingDir).sort()) {
|
|
611
|
+
if (!/^s\d+-\d+\.[a-z0-9]+$/i.test(f)) continue;
|
|
612
|
+
const name = `${b.stem}-${f.toLowerCase()}`;
|
|
613
|
+
renameSync(join2(b.stagingDir, f), join2(b.imagesDir, name));
|
|
614
|
+
b.manifest.add(name);
|
|
615
|
+
b.sourceMap.set(f, `images/${name}`);
|
|
616
|
+
}
|
|
617
|
+
}
|
|
618
|
+
function rewriteImageLinks(md, sourceMap) {
|
|
619
|
+
return md.replace(IMG_LINK_RE, (whole, mdAngle, mdPlain, htmlDouble, htmlSingle, htmlUnquoted) => {
|
|
620
|
+
const target = imageTarget([whole, mdAngle, mdPlain, htmlDouble, htmlSingle, htmlUnquoted]);
|
|
621
|
+
const dest = sourceMap.get(target) ?? sourceMap.get(target.replace(/^\.\//, ""));
|
|
622
|
+
return dest ? whole.replace(mdAngle === void 0 ? target : `<${mdAngle}>`, dest) : whole;
|
|
623
|
+
});
|
|
575
624
|
}
|
|
576
|
-
function
|
|
577
|
-
|
|
625
|
+
function validateImageLinks(md, manifest) {
|
|
626
|
+
for (const m of md.matchAll(IMG_LINK_RE)) {
|
|
627
|
+
const target = imageTarget(m);
|
|
628
|
+
if (!target.startsWith("images/") || !manifest.has(target.slice("images/".length))) throw new Error(`unexpected image reference in output: ${target}`);
|
|
629
|
+
}
|
|
578
630
|
}
|
|
579
|
-
function
|
|
580
|
-
const
|
|
631
|
+
function commitBundle(b, markdown) {
|
|
632
|
+
const tmp = `${b.mdPath}.tmp`;
|
|
633
|
+
writeFileSync2(tmp, markdown, "utf8");
|
|
634
|
+
renameSync(tmp, b.mdPath);
|
|
635
|
+
try {
|
|
636
|
+
fs.rmSync(b.stagingDir, { recursive: true, force: true });
|
|
637
|
+
} catch {
|
|
638
|
+
}
|
|
639
|
+
try {
|
|
640
|
+
fs.rmSync(b.lockPath, { force: true });
|
|
641
|
+
} catch {
|
|
642
|
+
}
|
|
643
|
+
}
|
|
644
|
+
function abortBundle(b) {
|
|
645
|
+
for (const f of b.manifest) rmSync(join2(b.imagesDir, f), { force: true });
|
|
646
|
+
rmSync(`${b.mdPath}.tmp`, { force: true });
|
|
647
|
+
rmSync(b.stagingDir, { recursive: true, force: true });
|
|
648
|
+
rmSync(b.lockPath, { force: true });
|
|
649
|
+
}
|
|
650
|
+
function tempBundleRoot() {
|
|
651
|
+
const dir = join2(tmpdir2(), `pi-quiver-doc-to-md-${randomBytes(4).toString("hex")}`);
|
|
581
652
|
mkdirSync2(dir, { recursive: true });
|
|
582
|
-
|
|
583
|
-
const hash = createHash2("sha1").update(resolve(inputPath)).digest("hex").slice(0, 8);
|
|
584
|
-
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
585
|
-
return join2(dir, `${stamp}-${base}-${hash}.${ext}`);
|
|
653
|
+
return resolve(dir);
|
|
586
654
|
}
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
655
|
+
|
|
656
|
+
// lib/doc-to-md-handle.ts
|
|
657
|
+
var TITLE_MAX = 80;
|
|
658
|
+
var NOTE_MAX_LINES = 5;
|
|
659
|
+
var NOTE_MAX_CHARS = 200;
|
|
660
|
+
var META_MAX_CHARS = 120;
|
|
661
|
+
function formatSize2(bytes) {
|
|
662
|
+
if (bytes < 1024) return `${bytes}B`;
|
|
663
|
+
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)}KB`;
|
|
664
|
+
return `${(bytes / (1024 * 1024)).toFixed(1)}MB`;
|
|
592
665
|
}
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
666
|
+
var trunc = (s, max) => s.length > max ? `${s.slice(0, max - 3)}...` : s;
|
|
667
|
+
function compactRanges(nums, maxEntries = 20) {
|
|
668
|
+
const sorted = [...new Set(nums)].sort((a, b) => a - b);
|
|
669
|
+
const parts = [];
|
|
670
|
+
for (let i = 0; i < sorted.length; ) {
|
|
671
|
+
let j = i;
|
|
672
|
+
while (j + 1 < sorted.length && sorted[j + 1] === sorted[j] + 1) j++;
|
|
673
|
+
parts.push(j > i ? `${sorted[i]}-${sorted[j]}` : `${sorted[i]}`);
|
|
674
|
+
i = j + 1;
|
|
598
675
|
}
|
|
599
|
-
return
|
|
676
|
+
if (parts.length <= maxEntries) return parts.join(", ");
|
|
677
|
+
return `${parts.slice(0, maxEntries).join(", ")} (+${parts.length - maxEntries} more)`;
|
|
678
|
+
}
|
|
679
|
+
function scanOutline(md, max) {
|
|
680
|
+
const entries = [];
|
|
681
|
+
let total = 0, inFence = false;
|
|
682
|
+
md.split("\n").forEach((raw, i) => {
|
|
683
|
+
const line = raw.replace(/\r$/, "");
|
|
684
|
+
if (/^\s*(```|~~~)/.test(line)) {
|
|
685
|
+
inFence = !inFence;
|
|
686
|
+
return;
|
|
687
|
+
}
|
|
688
|
+
if (inFence) return;
|
|
689
|
+
const m = line.match(/^(#{1,6}) (.*)$/);
|
|
690
|
+
if (!m) return;
|
|
691
|
+
total++;
|
|
692
|
+
if (entries.length < max) entries.push({ line: i + 1, level: m[1].length, title: trunc(m[2].trim(), TITLE_MAX) });
|
|
693
|
+
});
|
|
694
|
+
return { entries, total };
|
|
695
|
+
}
|
|
696
|
+
function outlineLines(entries, total) {
|
|
697
|
+
if (entries.length === 0) return [];
|
|
698
|
+
const width = Math.max(5, Math.max(...entries.map((e) => `L${e.line}`.length)) + 2);
|
|
699
|
+
const out = ["Outline:", ...entries.map((e) => ` ${`L${e.line}`.padEnd(width)}${"#".repeat(e.level)} ${trunc(e.title, TITLE_MAX)}`)];
|
|
700
|
+
if (total > entries.length) out.push(` (+${total - entries.length} more)`);
|
|
701
|
+
return out;
|
|
702
|
+
}
|
|
703
|
+
function formatHandle(h) {
|
|
704
|
+
const lines = [`Saved-To: ${h.savedTo}`];
|
|
705
|
+
if (h.imagesDir && h.imageCount > 0) lines.push(`Images-Dir: ${h.imagesDir}`);
|
|
706
|
+
lines.push(`Type: ${h.type} Engine: ${h.engine} Tier: ${h.tier}`);
|
|
707
|
+
lines.push(`Page-Count: ${h.pageCount ?? "?"} Pages: ${h.pages ? compactRanges(h.pages) : "all"} Images: ${h.imageCount} Size: ${formatSize2(h.bytes)} / ${h.lines} lines`);
|
|
708
|
+
if (h.degraded) lines.push(`Degraded: ${h.degraded}`);
|
|
709
|
+
if (h.fallbackReason) lines.push(`Fallback-Reason: ${h.fallbackReason}`);
|
|
710
|
+
const fe = [];
|
|
711
|
+
if (h.failedPages.length) fe.push(`Failed-Pages: ${compactRanges(h.failedPages)}`);
|
|
712
|
+
if (h.emptyPages.length) fe.push(`Empty-Pages: ${compactRanges(h.emptyPages)}`);
|
|
713
|
+
if (fe.length) lines.push(fe.join(" "));
|
|
714
|
+
h.notes.slice(0, NOTE_MAX_LINES).forEach((n, i) => lines.push(`${i === 0 ? "Notes: " : " "}${trunc(n, NOTE_MAX_CHARS)}`));
|
|
715
|
+
lines.push(...outlineLines(h.outline, h.outlineTotal));
|
|
716
|
+
return lines.join("\n");
|
|
717
|
+
}
|
|
718
|
+
function formatInfoHandle(i, max) {
|
|
719
|
+
if (i.sheets) {
|
|
720
|
+
const lines2 = [`Type: ${i.type} Sheets: ${i.sheetsTotal}`];
|
|
721
|
+
for (const s of i.sheets.slice(0, max)) {
|
|
722
|
+
const dims = `rows=${s.rows ?? "?"} cols=${s.cols ?? "?"}`;
|
|
723
|
+
const hidden = s.hiddenRows || s.hiddenCols ? ` hiddenRows=${s.hiddenRows} hiddenCols=${s.hiddenCols}` : "";
|
|
724
|
+
lines2.push(` ${trunc(s.name, TITLE_MAX)} ${s.hidden ? "hidden " : ""}${dims}${hidden}`);
|
|
725
|
+
}
|
|
726
|
+
if (i.sheetsTotal > max) lines2.push(` (+${i.sheetsTotal - max} more)`);
|
|
727
|
+
return lines2.join("\n");
|
|
728
|
+
}
|
|
729
|
+
const lines = [`Type: ${i.type} Page-Count: ${i.pageCount ?? "?"} Backend: ${i.backend}`];
|
|
730
|
+
const meta = Object.entries(i.metadata).filter(([, v]) => v).map(([k, v]) => `${k[0].toUpperCase()}${k.slice(1)}: ${trunc(v, META_MAX_CHARS)}`);
|
|
731
|
+
if (meta.length) lines.push(meta.join(" "));
|
|
732
|
+
if (i.toc.length) {
|
|
733
|
+
lines.push("TOC:", ...i.toc.slice(0, max).map((t) => ` L${t.level} ${trunc(t.title, TITLE_MAX)} (p${t.page})`));
|
|
734
|
+
if (i.tocTotal > max) lines.push(` (+${i.tocTotal - max} more)`);
|
|
735
|
+
}
|
|
736
|
+
return lines.join("\n");
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
// lib/doc-to-md-options.ts
|
|
740
|
+
import { extname as extname2 } from "node:path";
|
|
741
|
+
var UsageError = class extends Error {
|
|
742
|
+
};
|
|
743
|
+
var MIN_PYMUPDF4LLM = "1.27.0";
|
|
744
|
+
var VERSION_RE = /^\d+(\.\d+)*$/;
|
|
745
|
+
var DOC_TO_MD_OPTIONS = [
|
|
746
|
+
{ key: "path", flag: null, type: "string", default: null, settable: false, help: "Local .pdf .docx .pptx .xlsx .xls file" },
|
|
747
|
+
{ key: "info", flag: "--info", type: "bool", default: false, settable: false, help: "Inspect only (page count, metadata, TOC or sheet inventory); no bundle" },
|
|
748
|
+
{ key: "pages", flag: "--pages", type: "pages", default: null, settable: false, help: 'Inclusive 1-based pages, e.g. "12-15" or "3,7,10-12" (PDF/DOCX/PPTX only); default all' },
|
|
749
|
+
{ key: "outputDir", flag: "--output-dir", type: "string", default: null, settable: false, help: "Bundle root for <stem>.md + images/; default a per-call temp dir" },
|
|
750
|
+
{ key: "overwrite", flag: "--overwrite", type: "bool", default: false, settable: false, help: "Replace an existing completed <stem>.md bundle" },
|
|
751
|
+
{ key: "primaryTimeoutMs", flag: "--primary-timeout", type: "int", default: 6e4, settable: true, env: "PI_DOC_TO_MD_CONVERT_TIMEOUT_MS", help: "pymupdf4llm tier; also the unpdf tier" },
|
|
752
|
+
{ key: "fallbackTimeoutMs", flag: "--fallback-timeout", type: "int", default: 3e4, settable: true, help: "PyMuPDF get_text tier; also info on PDF" },
|
|
753
|
+
{ key: "sofficeTimeoutMs", flag: "--soffice-timeout", type: "int", default: 12e4, settable: true, env: "PI_DOC_TO_MD_SOFFICE_TIMEOUT_MS", help: "DOCX/PPTX -> PDF via LibreOffice" },
|
|
754
|
+
{ key: "excelTimeoutMs", flag: "--excel-timeout", type: "int", default: 6e4, settable: true, help: "Excel child (both openpyxl loads); also info on Excel" },
|
|
755
|
+
{ key: "warmTimeoutMs", flag: "--warm-timeout", type: "int", default: 12e4, settable: true, env: "PI_DOC_TO_MD_WARM_TIMEOUT_MS", help: "Absolute backend discovery/bootstrap deadline (first call per process)" },
|
|
756
|
+
{ key: "pymupdfVersion", flag: "--pymupdf-version", type: "version", default: "1.27.2.3", settable: true, env: "PI_DOC_TO_MD_PYMUPDF_VERSION", help: "pymupdf4llm pin (>= 1.27.0)" },
|
|
757
|
+
{ key: "imageDpi", flag: "--image-dpi", type: "int", default: 150, settable: true, help: "Render DPI for page images" },
|
|
758
|
+
{ key: "imageFormat", flag: "--image-format", type: "enum", default: "png", settable: true, enumValues: ["png", "jpg"], help: "Rendered image format (embedded images keep their native extension)" },
|
|
759
|
+
{ key: "maxCellsPerSheet", flag: "--max-cells-per-sheet", type: "int", default: 5e4, settable: true, help: "rows x cols budget per worksheet" },
|
|
760
|
+
{ key: "maxOutputBytes", flag: "--max-output-bytes", type: "int", default: 2e7, settable: true, help: "Child stdout cap in bytes" },
|
|
761
|
+
{ key: "outlineMaxEntries", flag: "--outline-max-entries", type: "int", default: 40, settable: true, help: "Heading outline / TOC / sheet inventory cap in the handle" }
|
|
762
|
+
];
|
|
763
|
+
var TUNABLE_DEFAULTS = Object.fromEntries(
|
|
764
|
+
DOC_TO_MD_OPTIONS.filter((d) => d.settable).map((d) => [d.key, d.default])
|
|
765
|
+
);
|
|
766
|
+
function versionAtLeast(v, min) {
|
|
767
|
+
const a = v.split(".").map(Number), b = min.split(".").map(Number);
|
|
768
|
+
for (let i = 0; i < Math.max(a.length, b.length); i++) {
|
|
769
|
+
const x = a[i] ?? 0, y = b[i] ?? 0;
|
|
770
|
+
if (x !== y) return x > y;
|
|
771
|
+
}
|
|
772
|
+
return true;
|
|
773
|
+
}
|
|
774
|
+
function coerceValue(d, raw, fromString = false) {
|
|
775
|
+
switch (d.type) {
|
|
776
|
+
case "int": {
|
|
777
|
+
const n = fromString && typeof raw === "string" ? Number(raw) : raw;
|
|
778
|
+
return typeof n === "number" && Number.isInteger(n) && n > 0 ? { ok: true, value: n } : { ok: false, reason: "must be a positive integer" };
|
|
779
|
+
}
|
|
780
|
+
case "bool":
|
|
781
|
+
return typeof raw === "boolean" ? { ok: true, value: raw } : { ok: false, reason: "must be true or false" };
|
|
782
|
+
case "enum":
|
|
783
|
+
return typeof raw === "string" && d.enumValues.includes(raw) ? { ok: true, value: raw } : { ok: false, reason: `must be one of ${d.enumValues.join(", ")}` };
|
|
784
|
+
case "version":
|
|
785
|
+
if (typeof raw !== "string" || !VERSION_RE.test(raw)) return { ok: false, reason: "must be digits and dots" };
|
|
786
|
+
if (!versionAtLeast(raw, MIN_PYMUPDF4LLM)) return { ok: false, reason: `must be >= ${MIN_PYMUPDF4LLM}` };
|
|
787
|
+
return { ok: true, value: raw };
|
|
788
|
+
case "string":
|
|
789
|
+
return typeof raw === "string" && raw.length > 0 ? { ok: true, value: raw } : { ok: false, reason: "must be a non-empty string" };
|
|
790
|
+
case "pages":
|
|
791
|
+
return typeof raw === "string" ? { ok: true, value: parsePages(raw) } : { ok: false, reason: "must be a string" };
|
|
792
|
+
}
|
|
793
|
+
}
|
|
794
|
+
function coerceDocToMdSettings(raw, warn = console.warn) {
|
|
795
|
+
if (raw === null || typeof raw !== "object" || Array.isArray(raw)) return void 0;
|
|
796
|
+
const out = {};
|
|
797
|
+
for (const [k, v] of Object.entries(raw)) {
|
|
798
|
+
const d = DOC_TO_MD_OPTIONS.find((o) => o.key === k);
|
|
799
|
+
if (!d || !d.settable) {
|
|
800
|
+
warn(`pi-quiver: quiver.docToMd.${k} is not a tunable setting; ignored.`);
|
|
801
|
+
continue;
|
|
802
|
+
}
|
|
803
|
+
const c = coerceValue(d, v);
|
|
804
|
+
if (!c.ok) {
|
|
805
|
+
warn(`pi-quiver: quiver.docToMd.${k} ${c.reason}; ignored.`);
|
|
806
|
+
continue;
|
|
807
|
+
}
|
|
808
|
+
out[k] = c.value;
|
|
809
|
+
}
|
|
810
|
+
return out;
|
|
811
|
+
}
|
|
812
|
+
function parsePages(spec) {
|
|
813
|
+
const out = /* @__PURE__ */ new Set();
|
|
814
|
+
const parts = spec.split(",").map((s) => s.trim());
|
|
815
|
+
if (parts.length === 0 || parts.some((p) => p === "")) throw new UsageError(`invalid --pages "${spec}": expected e.g. "12-15" or "3,7,10-12"`);
|
|
816
|
+
for (const p of parts) {
|
|
817
|
+
const m = p.match(/^(\d+)(?:-(\d+))?$/);
|
|
818
|
+
if (!m) throw new UsageError(`invalid --pages "${spec}": bad token "${p}"`);
|
|
819
|
+
const a = Number(m[1]), b = m[2] === void 0 ? a : Number(m[2]);
|
|
820
|
+
if (a < 1 || b < a) throw new UsageError(`invalid --pages "${spec}": pages are 1-based and ranges ascend`);
|
|
821
|
+
for (let i = a; i <= b; i++) out.add(i);
|
|
822
|
+
}
|
|
823
|
+
return [...out].sort((x, y) => x - y);
|
|
824
|
+
}
|
|
825
|
+
function sanitizeStem(base) {
|
|
826
|
+
const s = base.replace(/[^A-Za-z0-9._-]+/g, "_");
|
|
827
|
+
return s.length ? s : "document";
|
|
828
|
+
}
|
|
829
|
+
var SUPPORTED = { ".pdf": "pdf", ".docx": "docx", ".pptx": "pptx", ".xlsx": "xlsx", ".xls": "xls" };
|
|
830
|
+
function classifyInput(filePath) {
|
|
831
|
+
const t = SUPPORTED[extname2(filePath).toLowerCase()];
|
|
832
|
+
if (!t) throw new Error(`Unsupported file type "${extname2(filePath) || "(none)"}"; supported: .pdf, .docx, .pptx, .xlsx, .xls`);
|
|
833
|
+
return t;
|
|
834
|
+
}
|
|
835
|
+
function resolveOptions(perCall, settings, env) {
|
|
836
|
+
const out = { path: perCall.path };
|
|
837
|
+
for (const d of DOC_TO_MD_OPTIONS) {
|
|
838
|
+
if (d.key === "path") continue;
|
|
839
|
+
let value = d.default;
|
|
840
|
+
if (d.env && env[d.env] !== void 0) {
|
|
841
|
+
const c = coerceValue(d, env[d.env], true);
|
|
842
|
+
if (!c.ok) throw new UsageError(`${d.env} ${c.reason} (got "${env[d.env]}")`);
|
|
843
|
+
value = c.value;
|
|
844
|
+
}
|
|
845
|
+
if (d.settable && settings[d.key] !== void 0) value = settings[d.key];
|
|
846
|
+
const pc = perCall[d.key];
|
|
847
|
+
if (pc !== void 0 && pc !== null) {
|
|
848
|
+
const c = coerceValue(d, pc);
|
|
849
|
+
if (!c.ok) throw new UsageError(`${d.flag} ${c.reason}`);
|
|
850
|
+
value = c.value;
|
|
851
|
+
}
|
|
852
|
+
out[d.key] = value;
|
|
853
|
+
}
|
|
854
|
+
const o = out;
|
|
855
|
+
if (o.info && (o.pages !== null || o.outputDir !== null || o.overwrite)) {
|
|
856
|
+
throw new UsageError("--info cannot be combined with --pages, --output-dir or --overwrite");
|
|
857
|
+
}
|
|
858
|
+
return o;
|
|
859
|
+
}
|
|
860
|
+
function renderHelp() {
|
|
861
|
+
const row = (d) => ` ${(d.flag ?? "<path>").padEnd(26)} ${d.help}${d.default !== null && d.key !== "info" && d.key !== "overwrite" ? ` (default ${d.default})` : ""}`;
|
|
862
|
+
return [
|
|
863
|
+
"Usage: pi-quiver doc-to-md [flags] <path>",
|
|
864
|
+
"",
|
|
865
|
+
"Per-call:",
|
|
866
|
+
...DOC_TO_MD_OPTIONS.filter((d) => !d.settable).map(row),
|
|
867
|
+
"",
|
|
868
|
+
"Tunables (also settable under quiver.docToMd in settings.json; per-call > settings > default):",
|
|
869
|
+
...DOC_TO_MD_OPTIONS.filter((d) => d.settable).map(row),
|
|
870
|
+
"",
|
|
871
|
+
"Result: a handle (Saved-To, Images-Dir, Page-Count, Outline ...). Read the Saved-To file for the Markdown.",
|
|
872
|
+
"Exit codes: 0 success, 1 runtime error, 2 usage error."
|
|
873
|
+
].join("\n");
|
|
874
|
+
}
|
|
875
|
+
|
|
876
|
+
// lib/doc-to-md-core.ts
|
|
877
|
+
var PACKAGE_PINS = { pymupdf4llm: TUNABLE_DEFAULTS.pymupdfVersion, openpyxl: "3.1.5", xlrd: "2.0.2", pillow: "12.3.0" };
|
|
878
|
+
var KILL_GRACE_MS = 2e3;
|
|
879
|
+
var VENV_DIR_NAME = "doc-to-md-venv-v2";
|
|
880
|
+
var LEGACY_VENV_DIR_NAME = "pymupdf-venv";
|
|
881
|
+
var STDERR_CAP = 1e6;
|
|
882
|
+
var OUTPUT_MAX_BYTES = 2e7;
|
|
883
|
+
function withArgs(cfg) {
|
|
884
|
+
return ["--with", `pymupdf4llm==${cfg.pymupdfVersion}`, "--with", `openpyxl==${PACKAGE_PINS.openpyxl}`, "--with", `xlrd==${PACKAGE_PINS.xlrd}`, "--with", `pillow==${PACKAGE_PINS.pillow}`];
|
|
885
|
+
}
|
|
886
|
+
function pipInstallArgs(cfg) {
|
|
887
|
+
return ["-m", "pip", "install", `pymupdf4llm==${cfg.pymupdfVersion}`, `openpyxl==${PACKAGE_PINS.openpyxl}`, `xlrd==${PACKAGE_PINS.xlrd}`, `pillow==${PACKAGE_PINS.pillow}`];
|
|
600
888
|
}
|
|
601
889
|
function warmArgs(cfg) {
|
|
602
|
-
return ["run",
|
|
890
|
+
return ["run", ...withArgs(cfg), "--python", "3.14", "python", "-c", "import pymupdf4llm, openpyxl, xlrd, PIL"];
|
|
603
891
|
}
|
|
604
|
-
function
|
|
605
|
-
return ["run",
|
|
892
|
+
function uvChildArgs(cfg, script, mode) {
|
|
893
|
+
return ["run", ...withArgs(cfg), "--python", "3.14", "python", script, mode];
|
|
894
|
+
}
|
|
895
|
+
function pythonChildArgs(script, mode) {
|
|
896
|
+
return [script, mode];
|
|
606
897
|
}
|
|
607
898
|
function soffArgs(src, profileDir, outDir) {
|
|
608
899
|
return [
|
|
@@ -623,20 +914,41 @@ function soffArgs(src, profileDir, outDir) {
|
|
|
623
914
|
src
|
|
624
915
|
];
|
|
625
916
|
}
|
|
917
|
+
var LIVE = /* @__PURE__ */ new Set();
|
|
918
|
+
function killTree(child) {
|
|
919
|
+
if (child.pid === void 0) return;
|
|
920
|
+
try {
|
|
921
|
+
if (process.platform === "win32") spawn("taskkill", ["/T", "/F", "/PID", String(child.pid)], { stdio: "ignore" });
|
|
922
|
+
else process.kill(-child.pid, "SIGKILL");
|
|
923
|
+
} catch {
|
|
924
|
+
try {
|
|
925
|
+
child.kill("SIGKILL");
|
|
926
|
+
} catch {
|
|
927
|
+
}
|
|
928
|
+
}
|
|
929
|
+
}
|
|
930
|
+
process.on("exit", () => {
|
|
931
|
+
for (const c of LIVE) killTree(c);
|
|
932
|
+
});
|
|
626
933
|
async function runCapped(cmd, args, opts) {
|
|
627
934
|
return new Promise((resolveP) => {
|
|
628
935
|
let settled = false;
|
|
629
936
|
let timedOut = false;
|
|
630
937
|
let capped = false;
|
|
938
|
+
let timer;
|
|
939
|
+
let graceTimer;
|
|
631
940
|
const stdoutChunks = [];
|
|
632
941
|
const stderrChunks = [];
|
|
633
942
|
let stdoutBytes = 0;
|
|
634
943
|
let stderrBytes = 0;
|
|
635
|
-
const child = spawn(cmd, args, { env: opts.env ?? process.env });
|
|
944
|
+
const child = spawn(cmd, args, { env: opts.env ?? process.env, detached: process.platform !== "win32", stdio: ["pipe", "pipe", "pipe"] });
|
|
945
|
+
LIVE.add(child);
|
|
636
946
|
const finish = (code) => {
|
|
637
947
|
if (settled) return;
|
|
638
948
|
settled = true;
|
|
949
|
+
LIVE.delete(child);
|
|
639
950
|
clearTimeout(timer);
|
|
951
|
+
clearTimeout(graceTimer);
|
|
640
952
|
opts.signal?.removeEventListener("abort", onAbort);
|
|
641
953
|
resolveP({
|
|
642
954
|
stdout: Buffer.concat(stdoutChunks).toString("utf8"),
|
|
@@ -646,17 +958,25 @@ async function runCapped(cmd, args, opts) {
|
|
|
646
958
|
capped
|
|
647
959
|
});
|
|
648
960
|
};
|
|
649
|
-
const kill = () =>
|
|
961
|
+
const kill = () => {
|
|
962
|
+
killTree(child);
|
|
963
|
+
graceTimer ??= setTimeout(() => finish(null), KILL_GRACE_MS);
|
|
964
|
+
};
|
|
650
965
|
const onAbort = () => {
|
|
651
966
|
kill();
|
|
652
967
|
};
|
|
653
968
|
if (opts.signal?.aborted) {
|
|
654
969
|
kill();
|
|
655
970
|
} else opts.signal?.addEventListener("abort", onAbort);
|
|
656
|
-
|
|
971
|
+
timer = setTimeout(() => {
|
|
657
972
|
timedOut = true;
|
|
658
973
|
kill();
|
|
659
974
|
}, opts.timeoutMs);
|
|
975
|
+
child.stdin.on("error", (err) => {
|
|
976
|
+
if (err.code !== "EPIPE") throw err;
|
|
977
|
+
});
|
|
978
|
+
child.stdin.write(opts.stdin ?? "");
|
|
979
|
+
child.stdin.end();
|
|
660
980
|
child.stdout.on("data", (chunk) => {
|
|
661
981
|
if (capped) return;
|
|
662
982
|
const remaining = opts.capBytes - stdoutBytes;
|
|
@@ -696,73 +1016,29 @@ async function runCapped(cmd, args, opts) {
|
|
|
696
1016
|
function findPackageRoot(startDir) {
|
|
697
1017
|
let dir = startDir;
|
|
698
1018
|
for (; ; ) {
|
|
699
|
-
if (
|
|
1019
|
+
if (existsSync2(join3(dir, "package.json"))) return dir;
|
|
700
1020
|
const parent = dirname(dir);
|
|
701
1021
|
if (parent === dir) throw new Error(`package.json not found walking up from ${startDir}`);
|
|
702
1022
|
dir = parent;
|
|
703
1023
|
}
|
|
704
1024
|
}
|
|
705
1025
|
function scriptPath() {
|
|
706
|
-
return
|
|
707
|
-
}
|
|
708
|
-
function formatSize2(bytes) {
|
|
709
|
-
if (bytes < 1024) {
|
|
710
|
-
return `${bytes}B`;
|
|
711
|
-
} else if (bytes < 1024 * 1024) {
|
|
712
|
-
return `${(bytes / 1024).toFixed(1)}KB`;
|
|
713
|
-
} else {
|
|
714
|
-
return `${(bytes / (1024 * 1024)).toFixed(1)}MB`;
|
|
715
|
-
}
|
|
716
|
-
}
|
|
717
|
-
async function convertViaUv(cfg, pdfPath, signal) {
|
|
718
|
-
const r = await runCapped("uv", convertArgs(cfg, scriptPath(), pdfPath), {
|
|
719
|
-
timeoutMs: cfg.convertTimeoutMs,
|
|
720
|
-
capBytes: OUTPUT_MAX_BYTES,
|
|
721
|
-
signal
|
|
722
|
-
});
|
|
723
|
-
if (r.timedOut || r.capped || r.code !== 0) {
|
|
724
|
-
throw new Error(`code=${r.code} timedOut=${r.timedOut} capped=${r.capped}: ${r.stderr.slice(0, 500)}`);
|
|
725
|
-
}
|
|
726
|
-
return r.stdout;
|
|
727
|
-
}
|
|
728
|
-
async function convertViaPython(cfg, exe, pdfPath, signal) {
|
|
729
|
-
const r = await runCapped(exe, pythonConvertArgs(scriptPath(), pdfPath), {
|
|
730
|
-
timeoutMs: cfg.convertTimeoutMs,
|
|
731
|
-
capBytes: OUTPUT_MAX_BYTES,
|
|
732
|
-
signal
|
|
733
|
-
});
|
|
734
|
-
if (r.timedOut || r.capped || r.code !== 0) {
|
|
735
|
-
throw new Error(`code=${r.code} timedOut=${r.timedOut} capped=${r.capped}: ${r.stderr.slice(0, 500)}`);
|
|
736
|
-
}
|
|
737
|
-
return r.stdout;
|
|
738
|
-
}
|
|
739
|
-
async function convertViaUnpdf(pdfPath, cfg, signal) {
|
|
740
|
-
if (signal?.aborted) throw new Error("aborted");
|
|
741
|
-
const buf = readFileSync(pdfPath);
|
|
742
|
-
const work = (async () => {
|
|
743
|
-
const pdf = await getDocumentProxy(new Uint8Array(buf));
|
|
744
|
-
const { text } = await extractText(pdf, { mergePages: false });
|
|
745
|
-
const md = pagesToMarkdown(text);
|
|
746
|
-
if (Buffer.byteLength(md, "utf8") > OUTPUT_MAX_BYTES) throw new Error("unpdf output exceeded cap");
|
|
747
|
-
return md;
|
|
748
|
-
})();
|
|
749
|
-
let timer;
|
|
750
|
-
const timeout = new Promise((_, rej) => {
|
|
751
|
-
timer = setTimeout(() => rej(new Error("unpdf conversion timed out")), cfg.convertTimeoutMs);
|
|
752
|
-
});
|
|
753
|
-
try {
|
|
754
|
-
return await Promise.race([work, timeout]);
|
|
755
|
-
} finally {
|
|
756
|
-
clearTimeout(timer);
|
|
757
|
-
}
|
|
1026
|
+
return join3(findPackageRoot(dirname(fileURLToPath(import.meta.url))), "scripts", "doc_to_md.py");
|
|
758
1027
|
}
|
|
759
1028
|
var PROBE_PROGRAM = `import sys
|
|
760
1029
|
print("PY", sys.version_info[0], sys.version_info[1])
|
|
761
1030
|
try:
|
|
762
|
-
|
|
763
|
-
|
|
1031
|
+
# pymupdf4llm.__version__ >= 1.27.0
|
|
1032
|
+
import pymupdf, pymupdf4llm
|
|
1033
|
+
v = tuple(int(x) for x in pymupdf4llm.__version__.split(".")[:3])
|
|
1034
|
+
print("PDF", "yes" if v >= (1, 27, 0) else "no")
|
|
764
1035
|
except Exception:
|
|
765
|
-
print("
|
|
1036
|
+
print("PDF", "no")
|
|
1037
|
+
try:
|
|
1038
|
+
import openpyxl, xlrd, PIL
|
|
1039
|
+
print("XLSX", "yes")
|
|
1040
|
+
except Exception:
|
|
1041
|
+
print("XLSX", "no")
|
|
766
1042
|
`;
|
|
767
1043
|
var PROBE_TIMEOUT_MS = 5e3;
|
|
768
1044
|
function probeArgs() {
|
|
@@ -770,33 +1046,39 @@ function probeArgs() {
|
|
|
770
1046
|
}
|
|
771
1047
|
var PYTHON_CANDIDATES = ["python3", "python"];
|
|
772
1048
|
function parseProbeOutput(stdout) {
|
|
773
|
-
const m = stdout.match(/^PY (\d+) (\d+)\r?\
|
|
774
|
-
|
|
775
|
-
return { major: Number(m[1]), minor: Number(m[2]), pkg: m[3] === "none" ? null : m[3] };
|
|
1049
|
+
const m = stdout.match(/^PY (\d+) (\d+)\r?\nPDF (yes|no)\r?\nXLSX (yes|no)\s*$/);
|
|
1050
|
+
return m ? { major: Number(m[1]), minor: Number(m[2]), pdf: m[3] === "yes", xlsx: m[4] === "yes" } : null;
|
|
776
1051
|
}
|
|
777
1052
|
function meetsFloor(p) {
|
|
778
1053
|
return p.major > 3 || p.major === 3 && p.minor >= 12;
|
|
779
1054
|
}
|
|
780
1055
|
function cacheDir(platform, env, home) {
|
|
781
|
-
if (platform === "win32") return
|
|
782
|
-
if (platform === "darwin") return
|
|
783
|
-
return
|
|
1056
|
+
if (platform === "win32") return join3(env.LOCALAPPDATA ?? join3(home, "AppData", "Local"), "pi-quiver");
|
|
1057
|
+
if (platform === "darwin") return join3(home, "Library", "Caches", "pi-quiver");
|
|
1058
|
+
return join3(env.XDG_CACHE_HOME ?? join3(home, ".cache"), "pi-quiver");
|
|
784
1059
|
}
|
|
785
1060
|
function venvPython(venvDir, platform) {
|
|
786
|
-
return platform === "win32" ?
|
|
787
|
-
}
|
|
788
|
-
function pythonConvertArgs(script, pdfPath) {
|
|
789
|
-
return [script, pdfPath];
|
|
1061
|
+
return platform === "win32" ? join3(venvDir, "Scripts", "python.exe") : join3(venvDir, "bin", "python");
|
|
790
1062
|
}
|
|
791
1063
|
var tail = (s) => s.slice(-500).trim();
|
|
792
1064
|
async function resolveBackend(cfg, deps, signal) {
|
|
793
1065
|
if (signal?.aborted) throw new Error("aborted");
|
|
794
|
-
const
|
|
1066
|
+
const deadline = deps.now() + cfg.warmTimeoutMs;
|
|
1067
|
+
const left = () => Math.max(1, deadline - deps.now());
|
|
1068
|
+
const expired = (tmp) => {
|
|
1069
|
+
if (deadline - deps.now() > 0) return null;
|
|
1070
|
+
if (tmp) deps.rmrf(tmp);
|
|
1071
|
+
return { kind: "none", reason: `backend discovery exceeded warmTimeoutMs (${cfg.warmTimeoutMs}ms) - raise warmTimeoutMs, or install uv` };
|
|
1072
|
+
};
|
|
1073
|
+
const warm = await deps.run("uv", warmArgs(cfg), { timeoutMs: left(), capBytes: OUTPUT_MAX_BYTES, env: deps.env, signal });
|
|
795
1074
|
if (signal?.aborted) throw new Error("aborted");
|
|
796
|
-
if (warm.code === 0 && !warm.timedOut) return { kind: "uv" };
|
|
1075
|
+
if (warm.code === 0 && !warm.timedOut) return { kind: "uv", pdf: true, xlsx: true };
|
|
797
1076
|
const uvAbsent = warm.code === null && !warm.timedOut;
|
|
798
|
-
const
|
|
799
|
-
|
|
1077
|
+
const isDeadline = (result) => result !== null && "kind" in result;
|
|
1078
|
+
const probe = async (exe, tmp) => {
|
|
1079
|
+
const timeout = expired(tmp);
|
|
1080
|
+
if (timeout) return timeout;
|
|
1081
|
+
const r = await deps.run(exe, probeArgs(), { timeoutMs: Math.min(PROBE_TIMEOUT_MS, left()), capBytes: 4e3, env: deps.env, signal });
|
|
800
1082
|
if (signal?.aborted) throw new Error("aborted");
|
|
801
1083
|
if (r.code !== 0 || r.timedOut) return null;
|
|
802
1084
|
return parseProbeOutput(r.stdout);
|
|
@@ -804,28 +1086,33 @@ async function resolveBackend(cfg, deps, signal) {
|
|
|
804
1086
|
let eligible = null;
|
|
805
1087
|
for (const exe of PYTHON_CANDIDATES) {
|
|
806
1088
|
const p = await probe(exe);
|
|
1089
|
+
if (isDeadline(p)) return p;
|
|
807
1090
|
if (!p || !meetsFloor(p)) continue;
|
|
808
|
-
if (p.
|
|
1091
|
+
if (p.pdf) return { kind: "python", exe, pdf: true, xlsx: p.xlsx };
|
|
809
1092
|
eligible ??= { exe, version: `${p.major}.${p.minor}` };
|
|
810
1093
|
}
|
|
811
|
-
const venvDir =
|
|
1094
|
+
const venvDir = join3(deps.cacheRoot, VENV_DIR_NAME);
|
|
812
1095
|
const venvExe = venvPython(venvDir, deps.platform);
|
|
813
1096
|
const cached = await probe(venvExe);
|
|
814
|
-
if (
|
|
1097
|
+
if (isDeadline(cached)) return cached;
|
|
1098
|
+
if (cached && meetsFloor(cached) && cached.pdf && cached.xlsx) return { kind: "venv", exe: venvExe, pdf: true, xlsx: true };
|
|
815
1099
|
if (eligible) {
|
|
816
1100
|
const recheck = await probe(venvExe);
|
|
817
|
-
if (
|
|
1101
|
+
if (isDeadline(recheck)) return recheck;
|
|
1102
|
+
if (recheck && meetsFloor(recheck) && recheck.pdf && recheck.xlsx) return { kind: "venv", exe: venvExe, pdf: true, xlsx: true };
|
|
818
1103
|
const tmp = `${venvDir}.tmp-${deps.pid}`;
|
|
819
|
-
const deadline = deps.now() + cfg.warmTimeoutMs;
|
|
820
|
-
const left = () => Math.max(1, deadline - deps.now());
|
|
821
1104
|
const bootFail = (stderr) => {
|
|
822
1105
|
deps.rmrf(tmp);
|
|
823
1106
|
return { kind: "none", reason: `python ${eligible.version} found but venv bootstrap failed: ${tail(stderr)} - install python3-venv, or uv` };
|
|
824
1107
|
};
|
|
1108
|
+
const mkTimeout = expired(tmp);
|
|
1109
|
+
if (mkTimeout) return mkTimeout;
|
|
825
1110
|
const mk = await deps.run(eligible.exe, ["-m", "venv", tmp], { timeoutMs: left(), capBytes: OUTPUT_MAX_BYTES, env: deps.env, signal });
|
|
826
1111
|
if (signal?.aborted) throw new Error("aborted");
|
|
827
1112
|
if (mk.code !== 0 || mk.timedOut) return bootFail(mk.stderr);
|
|
828
|
-
const
|
|
1113
|
+
const pipTimeout = expired(tmp);
|
|
1114
|
+
if (pipTimeout) return pipTimeout;
|
|
1115
|
+
const pip = await deps.run(venvPython(tmp, deps.platform), pipInstallArgs(cfg), { timeoutMs: left(), capBytes: OUTPUT_MAX_BYTES, env: deps.env, signal });
|
|
829
1116
|
if (signal?.aborted) throw new Error("aborted");
|
|
830
1117
|
if (pip.code !== 0 || pip.timedOut) return bootFail(pip.stderr);
|
|
831
1118
|
const publish = () => {
|
|
@@ -836,14 +1123,21 @@ async function resolveBackend(cfg, deps, signal) {
|
|
|
836
1123
|
return false;
|
|
837
1124
|
}
|
|
838
1125
|
};
|
|
839
|
-
if (publish())
|
|
840
|
-
|
|
841
|
-
|
|
1126
|
+
if (publish()) {
|
|
1127
|
+
deps.rmrf(join3(deps.cacheRoot, LEGACY_VENV_DIR_NAME));
|
|
1128
|
+
return { kind: "venv", exe: venvExe, pdf: true, xlsx: true };
|
|
1129
|
+
}
|
|
1130
|
+
const winner = await probe(venvExe, tmp);
|
|
1131
|
+
if (isDeadline(winner)) return winner;
|
|
1132
|
+
if (winner && meetsFloor(winner) && winner.pdf && winner.xlsx) {
|
|
842
1133
|
deps.rmrf(tmp);
|
|
843
|
-
return { kind: "venv", exe: venvExe,
|
|
1134
|
+
return { kind: "venv", exe: venvExe, pdf: true, xlsx: true };
|
|
844
1135
|
}
|
|
845
1136
|
deps.rmrf(venvDir);
|
|
846
|
-
if (publish())
|
|
1137
|
+
if (publish()) {
|
|
1138
|
+
deps.rmrf(join3(deps.cacheRoot, LEGACY_VENV_DIR_NAME));
|
|
1139
|
+
return { kind: "venv", exe: venvExe, pdf: true, xlsx: true };
|
|
1140
|
+
}
|
|
847
1141
|
return bootFail("rename after competing bootstrap");
|
|
848
1142
|
}
|
|
849
1143
|
return uvAbsent ? { kind: "none", reason: "uv not found; no python >= 3.12 on PATH - install uv, or Python 3.12+" } : { kind: "none", reason: `uv warm-up failed: ${tail(warm.stderr)}; no python >= 3.12 on PATH` };
|
|
@@ -855,8 +1149,8 @@ function realDeps() {
|
|
|
855
1149
|
cacheRoot: cacheDir(process.platform, process.env, homedir()),
|
|
856
1150
|
platform: process.platform,
|
|
857
1151
|
pid: process.pid,
|
|
858
|
-
rename:
|
|
859
|
-
rmrf: (p) =>
|
|
1152
|
+
rename: renameSync2,
|
|
1153
|
+
rmrf: (p) => rmSync2(p, { recursive: true, force: true }),
|
|
860
1154
|
now: Date.now,
|
|
861
1155
|
env: process.env
|
|
862
1156
|
};
|
|
@@ -871,28 +1165,28 @@ function getBackend(cfg, deps, signal) {
|
|
|
871
1165
|
}
|
|
872
1166
|
return backendPromise;
|
|
873
1167
|
}
|
|
874
|
-
async function convertOffice(
|
|
875
|
-
const profileDir = mkdtempSync(
|
|
876
|
-
const outDir = mkdtempSync(
|
|
1168
|
+
async function convertOffice(sofficeTimeoutMs, src, signal, run = runCapped) {
|
|
1169
|
+
const profileDir = mkdtempSync(join3(tmpdir3(), "pi-doc-soffice-prof-"));
|
|
1170
|
+
const outDir = mkdtempSync(join3(tmpdir3(), "pi-doc-soffice-out-"));
|
|
877
1171
|
const cleanup = () => {
|
|
878
1172
|
for (const d of [profileDir, outDir]) {
|
|
879
1173
|
try {
|
|
880
|
-
|
|
1174
|
+
rmSync2(d, { recursive: true, force: true });
|
|
881
1175
|
} catch {
|
|
882
1176
|
}
|
|
883
1177
|
}
|
|
884
1178
|
};
|
|
885
1179
|
try {
|
|
886
1180
|
const env = { ...process.env, SAL_USE_VCLPLUGIN: "svp", OOO_DISABLE_RECOVERY: "1", SAL_NO_MOUSEGRABS: "1" };
|
|
887
|
-
const r = await run("soffice", soffArgs(src, profileDir, outDir), { timeoutMs:
|
|
1181
|
+
const r = await run("soffice", soffArgs(src, profileDir, outDir), { timeoutMs: sofficeTimeoutMs, capBytes: OUTPUT_MAX_BYTES, env, signal });
|
|
888
1182
|
if (signal?.aborted) throw new Error("aborted");
|
|
889
1183
|
if (r.code === null && !r.timedOut) {
|
|
890
1184
|
throw new Error("LibreOffice (soffice) is required to convert .docx/.pptx but was not found on PATH. Install LibreOffice or convert the file to PDF first.");
|
|
891
1185
|
}
|
|
892
1186
|
if (r.timedOut || r.code !== 0) throw new Error(`soffice failed (code=${r.code} timedOut=${r.timedOut}): ${r.stderr.slice(0, 500)}`);
|
|
893
1187
|
const base = basename(src).replace(/\.[^.]+$/, "");
|
|
894
|
-
const pdfPath =
|
|
895
|
-
const st =
|
|
1188
|
+
const pdfPath = join3(outDir, `${base}.pdf`);
|
|
1189
|
+
const st = statSync2(pdfPath, { throwIfNoEntry: false });
|
|
896
1190
|
if (!st || !st.isFile() || st.size === 0) {
|
|
897
1191
|
throw new Error("LibreOffice (soffice) ran but produced no usable PDF for this file. Ensure LibreOffice can open the document, or convert it to PDF manually first.");
|
|
898
1192
|
}
|
|
@@ -902,92 +1196,224 @@ async function convertOffice(cfg, src, signal, run = runCapped) {
|
|
|
902
1196
|
throw e;
|
|
903
1197
|
}
|
|
904
1198
|
}
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
1199
|
+
var DEGRADED_TEXT = "PyMuPDF text extraction - layout/tables not preserved";
|
|
1200
|
+
var DEGRADED_UNPDF = "unpdf text extraction - structure not preserved";
|
|
1201
|
+
var EXCEL_REMEDY = "Remedy: install uv, or pip install openpyxl xlrd pillow";
|
|
1202
|
+
function resolveUnpdfWorker(candidates, exists) {
|
|
1203
|
+
for (const candidate of candidates) if (exists(candidate)) return candidate;
|
|
1204
|
+
throw new Error("unpdf worker not found");
|
|
1205
|
+
}
|
|
1206
|
+
function unpdfWorkerPath() {
|
|
1207
|
+
const here = dirname(fileURLToPath(import.meta.url));
|
|
1208
|
+
return resolveUnpdfWorker([
|
|
1209
|
+
join3(findPackageRoot(here), "dist", "lib", "unpdf-worker.js"),
|
|
1210
|
+
join3(here, "unpdf-worker.js"),
|
|
1211
|
+
join3(here, "unpdf-worker.ts")
|
|
1212
|
+
], existsSync2);
|
|
1213
|
+
}
|
|
1214
|
+
async function runTierReal(mode, childOptions, _b, signal, timeoutMs, backend) {
|
|
1215
|
+
const cfg = { pymupdfVersion: String(childOptions.pymupdfVersion), warmTimeoutMs: 0 };
|
|
1216
|
+
let cmd, args;
|
|
1217
|
+
if (mode === "pdf-text" || mode === "info" && backend.kind === "none") {
|
|
1218
|
+
cmd = process.execPath;
|
|
1219
|
+
args = [unpdfWorkerPath(), mode];
|
|
1220
|
+
} else if (backend.kind === "uv") {
|
|
1221
|
+
cmd = "uv";
|
|
1222
|
+
args = uvChildArgs(cfg, scriptPath(), mode);
|
|
1223
|
+
} else if (backend.kind === "python" || backend.kind === "venv") {
|
|
1224
|
+
cmd = backend.exe;
|
|
1225
|
+
args = pythonChildArgs(scriptPath(), mode);
|
|
1226
|
+
} else return { ok: false, reason: backend.reason };
|
|
1227
|
+
const r = await runCapped(cmd, args, { timeoutMs, capBytes: Number(childOptions.maxOutputBytes), signal, stdin: JSON.stringify(childOptions) });
|
|
1228
|
+
if (signal?.aborted) return { ok: false, reason: "aborted" };
|
|
1229
|
+
if (r.timedOut) return { ok: false, reason: `timeout after ${timeoutMs}ms` };
|
|
1230
|
+
if (r.capped) return { ok: false, reason: "output exceeded maxOutputBytes" };
|
|
1231
|
+
const detail = r.stderr.slice(-300).trim();
|
|
1232
|
+
let json;
|
|
1233
|
+
try {
|
|
1234
|
+
json = JSON.parse(r.stdout);
|
|
1235
|
+
} catch {
|
|
1236
|
+
return r.code === 0 ? { ok: false, reason: "invalid-json" } : { ok: false, reason: `exit ${r.code ?? -1}`, ...detail ? { detail } : {} };
|
|
918
1237
|
}
|
|
1238
|
+
if (r.code === 3) return { ok: false, userError: json.error ?? "user error", ...json.pageCount !== void 0 ? { pageCount: json.pageCount } : {} };
|
|
1239
|
+
if (r.code !== 0) return { ok: false, reason: `exit ${r.code ?? -1}`, ...detail ? { detail } : {} };
|
|
1240
|
+
return { ok: true, json };
|
|
1241
|
+
}
|
|
1242
|
+
function detailSuffix(r) {
|
|
1243
|
+
return r.detail ? ` (${r.detail})` : "";
|
|
1244
|
+
}
|
|
1245
|
+
async function convertDocument(o, signal, seams) {
|
|
1246
|
+
const s = { backend: (c) => getBackend(c, void 0, signal), runTier: runTierReal, ...seams };
|
|
1247
|
+
const inputPath = resolve2(o.path);
|
|
1248
|
+
const st = statSync2(inputPath, { throwIfNoEntry: false });
|
|
1249
|
+
if (!st || !st.isFile()) throw new Error(`Not a readable file: ${o.path}`);
|
|
1250
|
+
const type = classifyInput(inputPath);
|
|
1251
|
+
const isExcel = type === "xlsx" || type === "xls";
|
|
1252
|
+
if (isExcel && o.pages) throw new Error("--pages does not apply to spreadsheets: worksheets have no stable page numbering");
|
|
1253
|
+
const backend = await s.backend({ pymupdfVersion: o.pymupdfVersion, warmTimeoutMs: o.warmTimeoutMs });
|
|
1254
|
+
if (isExcel && (backend.kind === "none" || !backend.xlsx)) throw new Error(`Excel conversion needs a Python backend with openpyxl, xlrd and pillow (${backend.kind === "none" ? backend.reason : `${backend.kind} lacks the Excel packages`}). ${EXCEL_REMEDY}`);
|
|
1255
|
+
const stem = sanitizeStem(basename(inputPath, extname3(inputPath)));
|
|
1256
|
+
const b = openBundle(o.outputDir ? resolve2(o.outputDir) : tempBundleRoot(), stem, o.overwrite);
|
|
1257
|
+
let office = null;
|
|
919
1258
|
try {
|
|
920
|
-
|
|
921
|
-
if (
|
|
922
|
-
|
|
1259
|
+
let pdfPath = inputPath;
|
|
1260
|
+
if (type === "docx" || type === "pptx") {
|
|
1261
|
+
office = await convertOffice(o.sofficeTimeoutMs, inputPath, signal);
|
|
1262
|
+
pdfPath = office.pdfPath;
|
|
923
1263
|
}
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
}
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
1264
|
+
const base = { path: pdfPath, pages: o.pages, stagingDir: b.stagingDir, imageDpi: o.imageDpi, imageFormat: o.imageFormat, maxCellsPerSheet: o.maxCellsPerSheet, maxOutputBytes: o.maxOutputBytes, pymupdfVersion: o.pymupdfVersion };
|
|
1265
|
+
let tier, engine, json, degraded = null, fallbackReason = null;
|
|
1266
|
+
if (isExcel) {
|
|
1267
|
+
const r = await s.runTier("xlsx", base, b, signal, o.excelTimeoutMs, backend);
|
|
1268
|
+
if (!r.ok) {
|
|
1269
|
+
if ("userError" in r) throw new Error(r.userError);
|
|
1270
|
+
const remedy = r.reason.startsWith("timeout after") || r.reason === "output exceeded maxOutputBytes" ? ". Remedy: raise excelTimeoutMs or lower maxCellsPerSheet" : "";
|
|
1271
|
+
throw new Error(`Excel conversion failed: ${r.reason}${detailSuffix(r)}${remedy}`);
|
|
1272
|
+
}
|
|
1273
|
+
publishSheetImages(b);
|
|
1274
|
+
tier = "excel";
|
|
1275
|
+
engine = type === "xls" ? "xlrd" : "openpyxl";
|
|
1276
|
+
json = r.json;
|
|
1277
|
+
} else if (backend.kind === "none") {
|
|
1278
|
+
const r = await s.runTier("pdf-text", base, b, signal, o.primaryTimeoutMs, backend);
|
|
1279
|
+
if (!r.ok) throw new Error("userError" in r ? r.userError : `Conversion failed: unpdf ${r.reason}${detailSuffix(r)}`);
|
|
1280
|
+
tier = "unpdf";
|
|
1281
|
+
engine = "unpdf";
|
|
1282
|
+
json = r.json;
|
|
1283
|
+
degraded = DEGRADED_UNPDF;
|
|
1284
|
+
} else {
|
|
1285
|
+
const p = await s.runTier("pdf-primary", base, b, signal, o.primaryTimeoutMs, backend);
|
|
1286
|
+
const kept = publishStaged(b);
|
|
1287
|
+
if (p.ok) {
|
|
1288
|
+
tier = "primary";
|
|
1289
|
+
engine = "pymupdf4llm";
|
|
1290
|
+
json = p.json;
|
|
1291
|
+
} else if ("userError" in p) throw new Error(p.userError);
|
|
1292
|
+
else {
|
|
1293
|
+
if (signal?.aborted) throw new Error("aborted");
|
|
1294
|
+
const keepPages = Object.fromEntries([...kept.entries()].map(([k, v]) => [String(k), v]));
|
|
1295
|
+
const f = await s.runTier("pdf-fallback", { ...base, keepPages }, b, signal, o.fallbackTimeoutMs, backend);
|
|
1296
|
+
publishStaged(b);
|
|
1297
|
+
if (!f.ok) throw new Error("userError" in f ? f.userError : `Conversion failed: primary ${p.reason}; fallback ${f.reason}${detailSuffix(f)}`);
|
|
1298
|
+
tier = "fallback";
|
|
1299
|
+
engine = "pymupdf-text";
|
|
1300
|
+
json = f.json;
|
|
1301
|
+
degraded = DEGRADED_TEXT;
|
|
1302
|
+
fallbackReason = `primary ${p.reason}`;
|
|
1303
|
+
}
|
|
942
1304
|
}
|
|
1305
|
+
let body = rewriteImageLinks(json.markdown ?? "", b.sourceMap);
|
|
1306
|
+
validateImageLinks(body, b.manifest);
|
|
1307
|
+
const head = [];
|
|
1308
|
+
if (degraded) head.push(`Degraded: ${degraded}`);
|
|
1309
|
+
if (fallbackReason) head.push(`Fallback-Reason: ${fallbackReason}`);
|
|
1310
|
+
if (json.failedPages?.length) head.push(`Failed pages: ${json.failedPages.map((f) => `${f.page} (${f.error})`).join("; ")}`);
|
|
1311
|
+
if (json.emptyPages?.length) head.push(`Empty pages: ${json.emptyPages.join(", ")}`);
|
|
1312
|
+
for (const n of json.notes ?? []) head.push(`Notes: ${n}`);
|
|
1313
|
+
const markdown = (head.length ? `${head.join("\n")}
|
|
1314
|
+
|
|
1315
|
+
` : "") + body;
|
|
1316
|
+
commitBundle(b, markdown);
|
|
1317
|
+
const outline = scanOutline(markdown, o.outlineMaxEntries);
|
|
1318
|
+
const details = { path: inputPath, backend: backend.kind, pymupdfVersion: o.pymupdfVersion, inputType: type, file: b.mdPath, outputDir: b.root, savedTo: b.mdPath, imagesDir: b.imagesDir, type, engine, tier, pageCount: json.pageCount ?? null, pages: o.pages, imageCount: b.manifest.size, bytes: Buffer.byteLength(markdown, "utf8"), lines: markdown.split("\n").length, degraded, fallbackReason, failedPages: (json.failedPages ?? []).map((f) => f.page), emptyPages: json.emptyPages ?? [], notes: json.notes ?? [], outline: outline.entries, outlineTotal: outline.total };
|
|
1319
|
+
return { output: formatHandle(details), details };
|
|
1320
|
+
} catch (e) {
|
|
1321
|
+
abortBundle(b);
|
|
1322
|
+
throw e;
|
|
943
1323
|
} finally {
|
|
944
|
-
cleanup
|
|
1324
|
+
office?.cleanup();
|
|
945
1325
|
}
|
|
946
1326
|
}
|
|
947
|
-
async function
|
|
948
|
-
const
|
|
949
|
-
const
|
|
950
|
-
|
|
1327
|
+
async function inspectDocument(o, signal, seams) {
|
|
1328
|
+
const s = { backend: (c) => getBackend(c, void 0, signal), runTier: runTierReal, ...seams };
|
|
1329
|
+
const inputPath = resolve2(o.path);
|
|
1330
|
+
const st = statSync2(inputPath, { throwIfNoEntry: false });
|
|
1331
|
+
if (!st || !st.isFile()) throw new Error(`Not a readable file: ${o.path}`);
|
|
951
1332
|
const type = classifyInput(inputPath);
|
|
952
|
-
const
|
|
953
|
-
const
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
1333
|
+
const isExcel = type === "xlsx" || type === "xls";
|
|
1334
|
+
const backend = await s.backend({ pymupdfVersion: o.pymupdfVersion, warmTimeoutMs: o.warmTimeoutMs });
|
|
1335
|
+
if (isExcel && (backend.kind === "none" || !backend.xlsx)) throw new Error(`Excel inspection needs a Python backend with openpyxl, xlrd and pillow. ${EXCEL_REMEDY}`);
|
|
1336
|
+
let office = null;
|
|
1337
|
+
try {
|
|
1338
|
+
let path = inputPath;
|
|
1339
|
+
if (type === "docx" || type === "pptx") {
|
|
1340
|
+
office = await convertOffice(o.sofficeTimeoutMs, inputPath, signal);
|
|
1341
|
+
path = office.pdfPath;
|
|
1342
|
+
}
|
|
1343
|
+
const r = await s.runTier("info", { path, maxOutputBytes: o.maxOutputBytes, pymupdfVersion: o.pymupdfVersion }, { stagingDir: "" }, signal, isExcel ? o.excelTimeoutMs : o.fallbackTimeoutMs, backend);
|
|
1344
|
+
if (!r.ok) {
|
|
1345
|
+
if ("userError" in r) throw new Error(r.userError);
|
|
1346
|
+
if (isExcel && (r.reason.startsWith("timeout after") || r.reason === "output exceeded maxOutputBytes")) throw new Error(`Excel inspection failed: ${r.reason}. Remedy: raise excelTimeoutMs or lower maxCellsPerSheet${detailSuffix(r)}`);
|
|
1347
|
+
throw new Error(`Inspection failed: ${r.reason}${detailSuffix(r)}`);
|
|
1348
|
+
}
|
|
1349
|
+
const toc = (r.json.toc ?? []).map(([level, title, page]) => ({ level, title, page }));
|
|
1350
|
+
const details = { type, backend: backend.kind, pageCount: r.json.pageCount ?? null, metadata: r.json.metadata ?? {}, toc: toc.slice(0, o.outlineMaxEntries), tocTotal: toc.length, sheets: r.json.sheets ? r.json.sheets.slice(0, o.outlineMaxEntries) : null, sheetsTotal: r.json.sheets?.length ?? 0 };
|
|
1351
|
+
return { output: formatInfoHandle(details, o.outlineMaxEntries), details };
|
|
1352
|
+
} finally {
|
|
1353
|
+
office?.cleanup();
|
|
964
1354
|
}
|
|
965
|
-
const file = spillToFile2(inputPath, body);
|
|
966
|
-
return {
|
|
967
|
-
output: compose([
|
|
968
|
-
...header,
|
|
969
|
-
`Body: ${formatSize2(bytes)} across ${lines} lines \u2014 written to file (too large to inline)`,
|
|
970
|
-
`Saved-To: ${file}`,
|
|
971
|
-
"",
|
|
972
|
-
"Read slices of this file with the read tool (offset/limit) or grep it; do not read the whole file unless you must. Markdown is grep-able by heading (^#).",
|
|
973
|
-
"",
|
|
974
|
-
"----- preview (first 60 lines) -----",
|
|
975
|
-
buildPreview2(body)
|
|
976
|
-
]),
|
|
977
|
-
details: { ...details, spilled: true, file }
|
|
978
|
-
};
|
|
979
1355
|
}
|
|
980
1356
|
|
|
981
1357
|
// bin/pi-quiver.ts
|
|
982
|
-
var USAGE = 'Usage: pi-quiver fetch <url> [--method GET|HEAD|POST] [--header "K: V"]... [--body <str>] [--raw] [--timeout-ms <n>]\n pi-quiver doc-to-md <path>';
|
|
983
|
-
function
|
|
984
|
-
if (
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
1358
|
+
var USAGE = 'Usage: pi-quiver fetch <url> [--method GET|HEAD|POST] [--header "K: V"]... [--body <str>] [--raw] [--timeout-ms <n>]\n pi-quiver doc-to-md [--info] [--pages <spec>] [--output-dir <dir>] [--overwrite] [tunable flags] <path> (--help for all flags)';
|
|
1359
|
+
function parseDocToMd(rest) {
|
|
1360
|
+
if (rest.includes("--help") || rest.includes("-h")) return { ok: true, cmd: "doc-to-md-help" };
|
|
1361
|
+
const perCall = {};
|
|
1362
|
+
let path;
|
|
1363
|
+
for (let i = 0; i < rest.length; i++) {
|
|
1364
|
+
const arg = rest[i];
|
|
1365
|
+
if (!arg.startsWith("--")) {
|
|
1366
|
+
if (path !== void 0) return { ok: false, error: `unexpected argument: ${arg}` };
|
|
1367
|
+
path = arg;
|
|
1368
|
+
continue;
|
|
1369
|
+
}
|
|
1370
|
+
const d = DOC_TO_MD_OPTIONS.find((o) => o.flag === arg);
|
|
1371
|
+
if (!d) return { ok: false, error: `unknown flag: ${arg}` };
|
|
1372
|
+
if (d.type === "bool") {
|
|
1373
|
+
perCall[d.key] = true;
|
|
1374
|
+
continue;
|
|
1375
|
+
}
|
|
1376
|
+
const value = rest[++i];
|
|
1377
|
+
if (value === void 0) return { ok: false, error: `${arg} requires a value` };
|
|
1378
|
+
if (d.type === "int") {
|
|
1379
|
+
const n = Number(value);
|
|
1380
|
+
if (!Number.isInteger(n) || n <= 0) return { ok: false, error: `invalid ${arg}: ${value}` };
|
|
1381
|
+
perCall[d.key] = n;
|
|
1382
|
+
} else perCall[d.key] = value;
|
|
990
1383
|
}
|
|
1384
|
+
if (!path) return { ok: false, error: "missing <path>" };
|
|
1385
|
+
return { ok: true, cmd: "doc-to-md", perCall: { ...perCall, path } };
|
|
1386
|
+
}
|
|
1387
|
+
function cliAgentDir(env) {
|
|
1388
|
+
const raw = env.PI_CODING_AGENT_DIR;
|
|
1389
|
+
const home = env.HOME ?? env.USERPROFILE ?? homedir2();
|
|
1390
|
+
if (raw && raw.length > 0) {
|
|
1391
|
+
if (raw === "~") return home;
|
|
1392
|
+
if (raw.startsWith("~/") || process.platform === "win32" && raw.startsWith("~\\")) return join4(home, raw.slice(2));
|
|
1393
|
+
return raw;
|
|
1394
|
+
}
|
|
1395
|
+
return join4(home, ".pi", "agent");
|
|
1396
|
+
}
|
|
1397
|
+
function readCliSettings(cwd, env, warn) {
|
|
1398
|
+
const out = {};
|
|
1399
|
+
for (const file of [join4(cliAgentDir(env), "settings.json"), join4(cwd, ".pi", "settings.json")]) {
|
|
1400
|
+
if (!existsSync3(file)) continue;
|
|
1401
|
+
let raw;
|
|
1402
|
+
try {
|
|
1403
|
+
raw = JSON.parse(readFileSync2(file, "utf8")).quiver?.docToMd;
|
|
1404
|
+
} catch {
|
|
1405
|
+
warn(`pi-quiver: ${file} is not valid JSON; ignored.`);
|
|
1406
|
+
continue;
|
|
1407
|
+
}
|
|
1408
|
+
if (raw === void 0) continue;
|
|
1409
|
+
const patch = coerceDocToMdSettings(raw, warn);
|
|
1410
|
+
if (patch) Object.assign(out, patch);
|
|
1411
|
+
else warn(`pi-quiver: "docToMd" in ${file} has an unrecognized value; ignored.`);
|
|
1412
|
+
}
|
|
1413
|
+
return out;
|
|
1414
|
+
}
|
|
1415
|
+
function parseCliArgs(argv) {
|
|
1416
|
+
if (argv[0] === "doc-to-md") return parseDocToMd(argv.slice(1));
|
|
991
1417
|
if (argv[0] !== "fetch") return { ok: false, error: `unknown command: ${argv[0] ?? "(none)"}` };
|
|
992
1418
|
const rest = argv.slice(1);
|
|
993
1419
|
let url;
|
|
@@ -1006,18 +1432,15 @@ function parseCliArgs(argv) {
|
|
|
1006
1432
|
const value = rest[++i];
|
|
1007
1433
|
if (value === void 0) return { ok: false, error: `${arg} requires a value` };
|
|
1008
1434
|
if (arg === "--method") {
|
|
1009
|
-
if (value !== "GET" && value !== "HEAD" && value !== "POST") {
|
|
1010
|
-
return { ok: false, error: `invalid --method: ${value}` };
|
|
1011
|
-
}
|
|
1435
|
+
if (value !== "GET" && value !== "HEAD" && value !== "POST") return { ok: false, error: `invalid --method: ${value}` };
|
|
1012
1436
|
method = value;
|
|
1013
1437
|
} else if (arg === "--header") {
|
|
1014
1438
|
const sep = value.indexOf(": ");
|
|
1015
1439
|
if (sep <= 0) return { ok: false, error: `malformed --header (expected "Key: Value"): ${value}` };
|
|
1016
1440
|
headers ??= {};
|
|
1017
1441
|
headers[value.slice(0, sep)] = value.slice(sep + 2);
|
|
1018
|
-
} else if (arg === "--body")
|
|
1019
|
-
|
|
1020
|
-
} else {
|
|
1442
|
+
} else if (arg === "--body") body = value;
|
|
1443
|
+
else {
|
|
1021
1444
|
const n = Number(value);
|
|
1022
1445
|
if (!Number.isFinite(n) || n <= 0) return { ok: false, error: `invalid --timeout-ms: ${value}` };
|
|
1023
1446
|
timeoutMs = n;
|
|
@@ -1045,10 +1468,30 @@ ${USAGE}
|
|
|
1045
1468
|
`);
|
|
1046
1469
|
return 2;
|
|
1047
1470
|
}
|
|
1471
|
+
if (parsed.cmd === "doc-to-md-help") {
|
|
1472
|
+
process.stdout.write(`${renderHelp()}
|
|
1473
|
+
`);
|
|
1474
|
+
return 0;
|
|
1475
|
+
}
|
|
1048
1476
|
if (parsed.cmd === "doc-to-md") {
|
|
1477
|
+
let o;
|
|
1049
1478
|
try {
|
|
1050
|
-
const
|
|
1051
|
-
|
|
1479
|
+
const settings = readCliSettings(process.cwd(), process.env, (m) => process.stderr.write(`${m}
|
|
1480
|
+
`));
|
|
1481
|
+
const pc = parsed.perCall;
|
|
1482
|
+
o = resolveOptions({ ...pc, path: resolve3(process.cwd(), pc.path), ...pc.outputDir ? { outputDir: resolve3(process.cwd(), pc.outputDir) } : {} }, settings, process.env);
|
|
1483
|
+
} catch (err) {
|
|
1484
|
+
if (err instanceof UsageError) {
|
|
1485
|
+
process.stderr.write(`${err.message}
|
|
1486
|
+
${USAGE}
|
|
1487
|
+
`);
|
|
1488
|
+
return 2;
|
|
1489
|
+
}
|
|
1490
|
+
throw err;
|
|
1491
|
+
}
|
|
1492
|
+
try {
|
|
1493
|
+
const r = o.info ? await inspectDocument(o) : await convertDocument(o);
|
|
1494
|
+
process.stdout.write(`${r.output}
|
|
1052
1495
|
`);
|
|
1053
1496
|
return 0;
|
|
1054
1497
|
} catch (err) {
|
|
@@ -1071,16 +1514,16 @@ ${USAGE}
|
|
|
1071
1514
|
function isMainEntry() {
|
|
1072
1515
|
if (!process.argv[1]) return false;
|
|
1073
1516
|
try {
|
|
1074
|
-
return realpathSync(fileURLToPath2(import.meta.url)) === realpathSync(
|
|
1517
|
+
return realpathSync(fileURLToPath2(import.meta.url)) === realpathSync(resolve3(process.argv[1]));
|
|
1075
1518
|
} catch {
|
|
1076
1519
|
return false;
|
|
1077
1520
|
}
|
|
1078
1521
|
}
|
|
1079
|
-
if (isMainEntry()) {
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
});
|
|
1083
|
-
}
|
|
1522
|
+
if (isMainEntry()) main().then((code) => {
|
|
1523
|
+
process.exitCode = code;
|
|
1524
|
+
});
|
|
1084
1525
|
export {
|
|
1085
|
-
|
|
1526
|
+
cliAgentDir,
|
|
1527
|
+
parseCliArgs,
|
|
1528
|
+
readCliSettings
|
|
1086
1529
|
};
|