pi-lightpanda-web 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/local-file.ts ADDED
@@ -0,0 +1,134 @@
1
+ import { open } from "node:fs/promises";
2
+ import type { FileHandle } from "node:fs/promises";
3
+ import { homedir } from "node:os";
4
+ import { join } from "node:path";
5
+ import { fileURLToPath } from "node:url";
6
+ import { pagePrefixedMarkdown } from "./html-to-md.ts";
7
+ import { extractPdfText } from "./pdf.ts";
8
+ import { pageText } from "./page-text.ts";
9
+
10
+ const BINARY_CHAR_RE = /[\x00-\x08\x0b\x0c\x0e-\x1a\x1c-\x1f\x7f-\x9f\ufffd]/g;
11
+ const MIN_BINARY_CHARS = 16;
12
+ const BINARY_CHAR_DIVISOR = 8;
13
+ const UTF32_LE_BOM = Buffer.from([0xff, 0xfe, 0x00, 0x00]);
14
+ const UTF32_BE_BOM = Buffer.from([0x00, 0x00, 0xfe, 0xff]);
15
+ const UTF16_LE_BOM = Buffer.from([0xff, 0xfe]);
16
+ const UTF16_BE_BOM = Buffer.from([0xfe, 0xff]);
17
+ const UTF8_BOM = Buffer.from([0xef, 0xbb, 0xbf]);
18
+ const PDF_MAGIC = Buffer.from("%PDF-");
19
+ const MAGIC_HEAD_BYTES = 1024;
20
+ const SKIPPED_MAGIC_PREFIX_BYTES = new Set([0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x20]);
21
+ const WINDOWS_PATH_RE = /^[a-zA-Z]:[\\/]/;
22
+ const HTML_FILE_EXTENSIONS = new Set([".htm", ".html", ".xht", ".xhtml"]);
23
+
24
+ export function parseLocalPath(url: string): string | null {
25
+ const trimmed = url.trim();
26
+ if (/^file:/i.test(trimmed)) {
27
+ try {
28
+ return fileURLToPath(trimmed);
29
+ } catch {
30
+ return "";
31
+ }
32
+ }
33
+ if (trimmed.startsWith("~/") || trimmed.startsWith("~\\")) return join(homedir(), trimmed.slice(2));
34
+ if (WINDOWS_PATH_RE.test(trimmed) || trimmed.startsWith("/") || /^\.{1,2}\//.test(trimmed)) return trimmed;
35
+ return null;
36
+ }
37
+
38
+ function localFileFailure(err: unknown): string {
39
+ return `Failed to read file: ${err instanceof Error ? err.message : String(err)}`;
40
+ }
41
+
42
+ function localFileContentType(filePath: string): string {
43
+ const dot = filePath.lastIndexOf(".");
44
+ return dot !== -1 && HTML_FILE_EXTENSIONS.has(filePath.slice(dot).toLowerCase()) ? "text/html" : "";
45
+ }
46
+
47
+ function looksBinary(text: string): boolean {
48
+ return (text.match(BINARY_CHAR_RE) ?? []).length > Math.max(MIN_BINARY_CHARS, Math.floor(text.length / BINARY_CHAR_DIVISOR));
49
+ }
50
+
51
+ function hasPdfMagic(data: Buffer): boolean {
52
+ const head = data.subarray(0, MAGIC_HEAD_BYTES);
53
+ let start = 0;
54
+ while (start < head.length && SKIPPED_MAGIC_PREFIX_BYTES.has(head[start])) start++;
55
+ for (const bom of [UTF32_LE_BOM, UTF32_BE_BOM, UTF16_LE_BOM, UTF16_BE_BOM, UTF8_BOM]) {
56
+ if (head.subarray(start).length >= bom.length && head.subarray(start, start + bom.length).equals(bom)) {
57
+ start += bom.length;
58
+ while (start < head.length && SKIPPED_MAGIC_PREFIX_BYTES.has(head[start])) start++;
59
+ break;
60
+ }
61
+ }
62
+ const magic = head.subarray(start);
63
+ return magic.length >= PDF_MAGIC.length && magic.subarray(0, PDF_MAGIC.length).equals(PDF_MAGIC);
64
+ }
65
+
66
+ function decodeUtf32(bytes: Buffer, littleEndian: boolean): string {
67
+ let out = "";
68
+ let i = 0;
69
+ if (bytes.length >= 4) {
70
+ const first = littleEndian ? bytes.readUInt32LE(0) : bytes.readUInt32BE(0);
71
+ if (first === 0xfeff) i = 4;
72
+ }
73
+ for (; i + 3 < bytes.length; i += 4) {
74
+ const value = littleEndian ? bytes.readUInt32LE(i) : bytes.readUInt32BE(i);
75
+ if (value === 0 || value > 0x10ffff || (value >= 0xd800 && value <= 0xdfff)) out += "\ufffd";
76
+ else out += String.fromCodePoint(value);
77
+ }
78
+ return out;
79
+ }
80
+
81
+ function decodeLocalText(bytes: Buffer): string {
82
+ if (bytes.length >= 4 && bytes.subarray(0, 4).equals(UTF32_LE_BOM)) return decodeUtf32(bytes, true);
83
+ if (bytes.length >= 4 && bytes.subarray(0, 4).equals(UTF32_BE_BOM)) return decodeUtf32(bytes, false);
84
+ if (bytes.length >= 2 && bytes.subarray(0, 2).equals(UTF16_LE_BOM)) {
85
+ return new TextDecoder("utf-16le", { fatal: false }).decode(bytes);
86
+ }
87
+ if (bytes.length >= 2 && bytes.subarray(0, 2).equals(UTF16_BE_BOM)) {
88
+ return new TextDecoder("utf-16be", { fatal: false }).decode(bytes);
89
+ }
90
+ return new TextDecoder("utf-8", { fatal: false }).decode(bytes);
91
+ }
92
+
93
+ export async function readLocalFile(
94
+ filePath: string,
95
+ options: { signal?: AbortSignal },
96
+ ): Promise<string> {
97
+ if (options.signal?.aborted) return "Failed to read file: cancelled.";
98
+ let handle: FileHandle;
99
+ try {
100
+ handle = await open(filePath, "r");
101
+ } catch (err) {
102
+ return localFileFailure(err);
103
+ }
104
+ try {
105
+ const stats = await handle.stat();
106
+ if (stats.isDirectory()) return `Failed to read file: ${filePath} is a directory.`;
107
+ const buffer = Buffer.alloc(Number(stats.size));
108
+ let read = 0;
109
+ while (read < buffer.length) {
110
+ const { bytesRead } = await handle.read(buffer, read, buffer.length - read, read);
111
+ if (!bytesRead) break;
112
+ read += bytesRead;
113
+ }
114
+ if (options.signal?.aborted) return "Failed to read file: cancelled.";
115
+ const body = read < buffer.length ? buffer.subarray(0, read) : buffer;
116
+ if (hasPdfMagic(body)) {
117
+ let pdfText: string;
118
+ try {
119
+ pdfText = await extractPdfText(body);
120
+ } catch {
121
+ return "(PDF content could not be read as text)";
122
+ }
123
+ if (!pdfText) pdfText = "(PDF contains no extractable text)";
124
+ return pageText(pdfText);
125
+ }
126
+ const text = decodeLocalText(body);
127
+ if (looksBinary(text)) return `(binary content, ${body.length} bytes; not readable as text)`;
128
+ const contentType = localFileContentType(filePath);
129
+ const rendered = contentType.includes("html") ? pagePrefixedMarkdown(text) : text.trim();
130
+ return pageText(rendered);
131
+ } finally {
132
+ await handle.close().catch(() => {});
133
+ }
134
+ }
package/package.json CHANGED
@@ -1,6 +1,84 @@
1
1
  {
2
2
  "name": "pi-lightpanda-web",
3
- "version": "0.0.0-stage",
4
- "stub": true,
5
- "description": "Temporary package placeholder for staged publishing"
6
- }
3
+ "version": "0.1.0",
4
+ "type": "module",
5
+ "description": "Pi Lightpanda Web: web_search and web_fetch powered entirely by the local Lightpanda browser",
6
+ "main": "index.ts",
7
+ "repository": {
8
+ "type": "git",
9
+ "url": "git+https://github.com/YuGiMob/pi-lightpanda-web.git"
10
+ },
11
+ "author": "YuGiMob",
12
+ "keywords": [
13
+ "pi-package",
14
+ "pi",
15
+ "coding-agent",
16
+ "extension",
17
+ "web-search",
18
+ "web-fetch",
19
+ "lightpanda",
20
+ "headless-browser"
21
+ ],
22
+ "license": "AGPL-3.0-only",
23
+ "files": [
24
+ "index.ts",
25
+ "web-search.ts",
26
+ "web-fetch.ts",
27
+ "lightpanda.ts",
28
+ "local-file.ts",
29
+ "pdf-fetch.ts",
30
+ "page-text.ts",
31
+ "proxy.ts",
32
+ "web-access.ts",
33
+ "html-to-md.ts",
34
+ "engines.ts",
35
+ "entities.ts",
36
+ "pdf.ts",
37
+ "cache.ts",
38
+ "settings.ts",
39
+ "agent-dir.ts",
40
+ "search-config.ts",
41
+ "README.md",
42
+ "LICENSE",
43
+ "scripts/install-lightpanda.sh"
44
+ ],
45
+ "pi": {
46
+ "extensions": [
47
+ "./index.ts"
48
+ ]
49
+ },
50
+ "peerDependencies": {
51
+ "@earendil-works/pi-coding-agent": "*",
52
+ "@earendil-works/pi-tui": "*",
53
+ "typebox": "*"
54
+ },
55
+ "engines": {
56
+ "node": ">=22.19.0"
57
+ },
58
+ "scripts": {
59
+ "check:attw": "attw --pack",
60
+ "check:package": "node -e \"const fs=require('fs');const pkg=require('./package.json');const listed=new Set(pkg.files);const bad=[];for(const f of pkg.files){if(!fs.existsSync(f))bad.push('missing file: '+f)}for(const f of fs.readdirSync('.').filter(f=>f.endsWith('.ts'))){if(!listed.has(f))bad.push('unlisted source: '+f)}if(bad.length){console.error(bad.join('\\n'));process.exit(1)}\"",
61
+ "check:publint": "publint",
62
+ "engine:eval": "node scripts/engine-eval.ts",
63
+ "lint": "oxlint .",
64
+ "prepublishOnly": "npm run typecheck && npm run lint && npm run test:unit && npm run check:package && npm run check:publint",
65
+ "test": "vitest run",
66
+ "test:live": "vitest run test/live.test.ts",
67
+ "test:unit": "vitest run --exclude **/live.test.ts",
68
+ "typecheck": "tsc --noEmit"
69
+ },
70
+ "devDependencies": {
71
+ "@earendil-works/pi-coding-agent": "^0.87.0",
72
+ "@earendil-works/pi-tui": "^0.87.0",
73
+ "@types/node": "^24.0.0",
74
+ "typebox": "^1.3.7",
75
+ "typescript": "~5.9.3",
76
+ "vitest": "^4.1.9",
77
+ "@arethetypeswrong/cli": "^0.18.5",
78
+ "oxlint": "^1.81.0",
79
+ "publint": "^0.3.24"
80
+ },
81
+ "optionalDependencies": {
82
+ "mupdf": "^1.28.0"
83
+ }
84
+ }
package/page-text.ts ADDED
@@ -0,0 +1,5 @@
1
+ export const EMPTY_PAGE_MESSAGE = "(page returned no readable text)";
2
+
3
+ export function pageText(text: string): string {
4
+ return text || EMPTY_PAGE_MESSAGE;
5
+ }
package/pdf-fetch.ts ADDED
@@ -0,0 +1,405 @@
1
+ import { lookup as dnsLookup } from "node:dns/promises";
2
+ import type { LookupAllOptions } from "node:dns";
3
+ import http from "node:http";
4
+ import https from "node:https";
5
+ import type { IncomingMessage } from "node:http";
6
+ import type { ClientRequest } from "node:http";
7
+ import { isIP } from "node:net";
8
+ import { brotliDecompressSync, gunzipSync, inflateRawSync, inflateSync } from "node:zlib";
9
+ import { extractPdfText } from "./pdf.ts";
10
+ import {
11
+ checkUrlAccess,
12
+ isPublicIp,
13
+ stripIpv6Brackets,
14
+ type WebsitePolicy,
15
+ } from "./web-access.ts";
16
+
17
+ const MAX_REQUESTS = 5;
18
+ const MAX_DNS_ATTEMPTS = 2;
19
+ const DNS_RETRY_BACKOFF_MS = 250;
20
+ const PDF_USER_AGENT =
21
+ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36";
22
+ const PDF_MAGIC = Buffer.from("%PDF-");
23
+ const CANCELLED_MESSAGE = "Failed to fetch URL: cancelled.";
24
+ const TIMEOUT_MESSAGE = "Failed to fetch URL: timed out.";
25
+ const TRANSIENT_DNS_ERROR_CODES = new Set([
26
+ "EAI_AGAIN",
27
+ "ETIMEDOUT",
28
+ "ECONNREFUSED",
29
+ "ENETUNREACH",
30
+ "EHOSTUNREACH",
31
+ ]);
32
+
33
+ class PdfCancelledError extends Error {
34
+ constructor() {
35
+ super("cancelled");
36
+ }
37
+ }
38
+
39
+ class PdfTimeoutError extends Error {
40
+ constructor() {
41
+ super("timed out");
42
+ }
43
+ }
44
+
45
+ export interface PdfResolvedHost {
46
+ ok: boolean;
47
+ reason: string;
48
+ ip: string;
49
+ family: number;
50
+ alternates?: { ip: string; family: number }[];
51
+ }
52
+
53
+ export interface PdfHopOptions {
54
+ url: URL;
55
+ pinnedIp: string;
56
+ family: number;
57
+ headers: Record<string, string>;
58
+ inactivityMs: number;
59
+ deadlineMs: number;
60
+ nowMs?: () => number;
61
+ signal?: AbortSignal;
62
+ }
63
+
64
+ export interface PdfHopResponse {
65
+ status: number;
66
+ headers: Record<string, string | string[] | undefined>;
67
+ body: Buffer;
68
+ encoding: string;
69
+ }
70
+
71
+ export interface PdfFetchSeams {
72
+ resolve?: (hostname: string, signal?: AbortSignal, allowPrivateAddresses?: boolean) => Promise<PdfResolvedHost>;
73
+ request?: (options: PdfHopOptions) => Promise<PdfHopResponse>;
74
+ }
75
+
76
+ export interface PdfFetchOptions {
77
+ timeoutMs?: number;
78
+ deadlineMs?: number;
79
+ nowMs?: () => number;
80
+ signal?: AbortSignal;
81
+ websitePolicy?: WebsitePolicy | null;
82
+ allowPrivateAddresses?: boolean;
83
+ seams?: PdfFetchSeams;
84
+ }
85
+
86
+ export type PdfFetchOutcome = { text: string } | { error: string } | null;
87
+
88
+ function isTransientDnsError(err: unknown): boolean {
89
+ if (!(err instanceof Error)) return false;
90
+ const code = (err as NodeJS.ErrnoException).code;
91
+ return typeof code === "string" && TRANSIENT_DNS_ERROR_CODES.has(code);
92
+ }
93
+
94
+ function sleepAbortable(ms: number, signal?: AbortSignal): Promise<void> {
95
+ return new Promise((resolve) => {
96
+ if (signal?.aborted) return resolve();
97
+ const timer = setTimeout(resolve, ms);
98
+ signal?.addEventListener(
99
+ "abort",
100
+ () => {
101
+ clearTimeout(timer);
102
+ resolve();
103
+ },
104
+ { once: true },
105
+ );
106
+ });
107
+ }
108
+
109
+ export async function resolveAndValidateHost(
110
+ hostname: string,
111
+ signal?: AbortSignal,
112
+ allowPrivateAddresses = true,
113
+ ): Promise<PdfResolvedHost> {
114
+ let lastError: unknown = null;
115
+ for (let attempt = 0; attempt < MAX_DNS_ATTEMPTS; attempt++) {
116
+ if (signal?.aborted) break;
117
+ if (attempt > 0) {
118
+ await sleepAbortable(DNS_RETRY_BACKOFF_MS, signal);
119
+ if (signal?.aborted) break;
120
+ }
121
+ try {
122
+ const lookupOptions: LookupAllOptions & { signal?: AbortSignal } = {
123
+ all: true,
124
+ verbatim: true,
125
+ signal,
126
+ };
127
+ const addresses = await dnsLookup(hostname, lookupOptions);
128
+ if (!addresses.length) {
129
+ return { ok: false, reason: `Failed to resolve host: no addresses for '${hostname}'`, ip: "", family: 0 };
130
+ }
131
+ for (const entry of addresses) {
132
+ if (!allowPrivateAddresses && !isPublicIp(entry.address)) {
133
+ return {
134
+ ok: false,
135
+ reason: `Blocked: refusing to fetch the non-public address ${entry.address}.`,
136
+ ip: "",
137
+ family: 0,
138
+ };
139
+ }
140
+ }
141
+ addresses.sort((a, b) => (a.family === 4 ? 0 : 1) - (b.family === 4 ? 0 : 1));
142
+ const first = addresses[0];
143
+ return {
144
+ ok: true,
145
+ reason: "",
146
+ ip: first.address,
147
+ family: first.family,
148
+ alternates: addresses.slice(1).map((entry) => ({ ip: entry.address, family: entry.family })),
149
+ };
150
+ } catch (err) {
151
+ lastError = err;
152
+ if (!isTransientDnsError(err)) break;
153
+ }
154
+ }
155
+ return {
156
+ ok: false,
157
+ reason: lastError === null ? "Failed to resolve host: cancelled" : `Failed to resolve host: ${lastError}`,
158
+ ip: "",
159
+ family: 0,
160
+ };
161
+ }
162
+
163
+ function headerValue(headers: Record<string, string | string[] | undefined>, name: string): string {
164
+ const value = headers[name];
165
+ return Array.isArray(value) ? (value[0] ?? "") : (value ?? "");
166
+ }
167
+
168
+ function statusError(status: number): string {
169
+ const reason = http.STATUS_CODES[status] ?? "";
170
+ return `Failed to fetch URL: HTTP ${status}${reason ? ` ${reason}` : ""}`;
171
+ }
172
+
173
+ export function looksLikePdfUrl(href: string): boolean {
174
+ let url: URL;
175
+ try {
176
+ url = new URL(href);
177
+ } catch {
178
+ return false;
179
+ }
180
+ const path = url.pathname.toLowerCase();
181
+ if (path.endsWith(".pdf") || /\/pdf\//.test(path)) return true;
182
+ for (const value of url.searchParams.values()) {
183
+ if (value.toLowerCase().endsWith(".pdf")) return true;
184
+ }
185
+ return (url.searchParams.get("format") ?? "").toLowerCase() === "pdf";
186
+ }
187
+
188
+ function hasPdfMagic(data: Buffer): boolean {
189
+ let start = 0;
190
+ while (start < data.length && [0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x20].includes(data[start])) start++;
191
+ for (const bom of [
192
+ Buffer.from([0xff, 0xfe, 0x00, 0x00]),
193
+ Buffer.from([0x00, 0x00, 0xfe, 0xff]),
194
+ Buffer.from([0xff, 0xfe]),
195
+ Buffer.from([0xfe, 0xff]),
196
+ Buffer.from([0xef, 0xbb, 0xbf]),
197
+ ]) {
198
+ if (data.subarray(start).length >= bom.length && data.subarray(start, start + bom.length).equals(bom)) {
199
+ start += bom.length;
200
+ while (start < data.length && [0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x20].includes(data[start])) start++;
201
+ break;
202
+ }
203
+ }
204
+ const magic = data.subarray(start);
205
+ return magic.length >= PDF_MAGIC.length && magic.subarray(0, PDF_MAGIC.length).equals(PDF_MAGIC);
206
+ }
207
+
208
+ function decodeBody(body: Buffer, encoding: string): Buffer {
209
+ if (!encoding) return body;
210
+ try {
211
+ if (encoding === "gzip" || encoding === "x-gzip") {
212
+ return gunzipSync(body);
213
+ }
214
+ if (encoding === "deflate") {
215
+ try {
216
+ return inflateSync(body);
217
+ } catch {
218
+ return inflateRawSync(body);
219
+ }
220
+ }
221
+ if (encoding === "br") {
222
+ return brotliDecompressSync(body);
223
+ }
224
+ } catch {
225
+ return body;
226
+ }
227
+ return body;
228
+ }
229
+
230
+ function requestPdfHop(options: PdfHopOptions): Promise<PdfHopResponse> {
231
+ return new Promise((resolve, reject) => {
232
+ const url = options.url;
233
+ const transport = url.protocol === "https:" ? https : http;
234
+ const port = url.port ? Number(url.port) : url.protocol === "https:" ? 443 : 80;
235
+ const hostname = stripIpv6Brackets(url.hostname);
236
+ const hostHeader = hostname.includes(":")
237
+ ? `[${hostname}]${url.port ? `:${url.port}` : ""}`
238
+ : hostname + (url.port ? `:${url.port}` : "");
239
+ const headers: Record<string, string> = {
240
+ "User-Agent": PDF_USER_AGENT,
241
+ Accept: "application/pdf,application/octet-stream;q=0.9,*/*;q=0.5",
242
+ "Accept-Encoding": "identity",
243
+ Host: hostHeader,
244
+ ...options.headers,
245
+ };
246
+ const requestOptions: https.RequestOptions = {
247
+ method: "GET",
248
+ host: hostname,
249
+ port,
250
+ path: url.pathname + url.search,
251
+ headers,
252
+ timeout: options.inactivityMs,
253
+ };
254
+ if (url.protocol === "https:" && !isIP(hostname)) requestOptions.servername = hostname;
255
+ requestOptions.lookup = ((_hostname: string, _lookupOptions: unknown, _callback: unknown) => {
256
+ const done = (typeof _lookupOptions === "function" ? _lookupOptions : _callback) as (
257
+ err: unknown,
258
+ address: unknown,
259
+ family?: unknown,
260
+ ) => void;
261
+ if ((_lookupOptions as { all?: boolean } | undefined)?.all) {
262
+ done(null, [{ address: options.pinnedIp, family: options.family }]);
263
+ } else {
264
+ done(null, options.pinnedIp, options.family);
265
+ }
266
+ }) as typeof requestOptions.lookup;
267
+ let settled = false;
268
+ let requestRef: ClientRequest | null = null;
269
+ let responseRef: IncomingMessage | null = null;
270
+ const now = options.nowMs ?? Date.now;
271
+ const finish = (action: () => void) => {
272
+ if (settled) return;
273
+ settled = true;
274
+ clearTimeout(deadlineTimer);
275
+ options.signal?.removeEventListener("abort", onAbort);
276
+ action();
277
+ };
278
+ const onAbort = () => {
279
+ responseRef?.destroy();
280
+ requestRef?.destroy();
281
+ finish(() => reject(new PdfCancelledError()));
282
+ };
283
+ const deadlineTimer = setTimeout(() => {
284
+ responseRef?.destroy();
285
+ requestRef?.destroy();
286
+ finish(() => reject(new PdfTimeoutError()));
287
+ }, Math.max(1, options.deadlineMs - now()));
288
+ options.signal?.addEventListener("abort", onAbort, { once: true });
289
+ if (options.signal?.aborted) {
290
+ onAbort();
291
+ return;
292
+ }
293
+ requestRef = transport.request(requestOptions, (response: IncomingMessage) => {
294
+ responseRef = response;
295
+ const chunks: Buffer[] = [];
296
+ const encoding = String(response.headers["content-encoding"] ?? "")
297
+ .split(",", 1)[0]
298
+ .trim()
299
+ .toLowerCase();
300
+ const finishResponse = () =>
301
+ finish(() =>
302
+ resolve({
303
+ status: response.statusCode ?? 0,
304
+ headers: response.headers as Record<string, string | string[] | undefined>,
305
+ body: Buffer.concat(chunks),
306
+ encoding,
307
+ }),
308
+ );
309
+ response.on("data", (chunk: Buffer) => {
310
+ if (settled) return;
311
+ chunks.push(chunk);
312
+ });
313
+ response.on("end", () => finishResponse());
314
+ response.on("aborted", () => finish(() => reject(new Error("response aborted"))));
315
+ response.on("close", () => {
316
+ if (!response.complete) finish(() => reject(new Error("response closed")));
317
+ });
318
+ response.on("error", (err) => finish(() => reject(err)));
319
+ });
320
+ requestRef.on("timeout", () => {
321
+ responseRef?.destroy();
322
+ requestRef?.destroy();
323
+ finish(() => reject(new PdfTimeoutError()));
324
+ });
325
+ requestRef.on("error", (err) => finish(() => reject(err)));
326
+ requestRef.end();
327
+ });
328
+ }
329
+
330
+ export async function fetchRemotePdf(
331
+ url: string,
332
+ options: PdfFetchOptions = {},
333
+ ): Promise<PdfFetchOutcome> {
334
+ const now = options.nowMs ?? Date.now;
335
+ const timeoutMs = options.timeoutMs ?? 60_000;
336
+ const deadline = options.deadlineMs ?? now() + timeoutMs;
337
+ const signal = options.signal;
338
+ const policy = options.websitePolicy ?? null;
339
+ const allowPrivateAddresses = options.allowPrivateAddresses ?? true;
340
+ const resolve = options.seams?.resolve ?? resolveAndValidateHost;
341
+ const request = options.seams?.request ?? requestPdfHop;
342
+ let currentUrl = url;
343
+ for (let hop = 0; hop < MAX_REQUESTS; hop++) {
344
+ if (signal?.aborted) return { error: CANCELLED_MESSAGE };
345
+ if (now() >= deadline) return { error: TIMEOUT_MESSAGE };
346
+ const access = checkUrlAccess(currentUrl, policy);
347
+ if (!access[0]) return { error: access[1] };
348
+ const hostname = access[2];
349
+ const resolved = await resolve(hostname, signal, allowPrivateAddresses);
350
+ if (!resolved.ok) return { error: resolved.reason };
351
+ const candidates = [{ ip: resolved.ip, family: resolved.family }, ...(resolved.alternates ?? [])];
352
+ let response: PdfHopResponse | null = null;
353
+ let failure: unknown = null;
354
+ for (const candidate of candidates) {
355
+ try {
356
+ response = await request({
357
+ url: new URL(currentUrl),
358
+ pinnedIp: candidate.ip,
359
+ family: candidate.family,
360
+ headers: {},
361
+ inactivityMs: Math.max(1, deadline - now()),
362
+ deadlineMs: deadline,
363
+ nowMs: now,
364
+ signal,
365
+ });
366
+ break;
367
+ } catch (err) {
368
+ if (err instanceof PdfCancelledError) return { error: CANCELLED_MESSAGE };
369
+ if (err instanceof PdfTimeoutError) return { error: TIMEOUT_MESSAGE };
370
+ if (signal?.aborted) return { error: CANCELLED_MESSAGE };
371
+ if (now() >= deadline) return { error: TIMEOUT_MESSAGE };
372
+ failure = err;
373
+ }
374
+ }
375
+ if (!response) {
376
+ return {
377
+ error: `Failed to fetch URL: ${failure instanceof Error ? failure.message : String(failure ?? "no transport could reach the target")}`,
378
+ };
379
+ }
380
+ if (response.status >= 300 && response.status < 400) {
381
+ const location = headerValue(response.headers, "location");
382
+ if (!location) return { error: "Failed to fetch URL: the redirect is missing a Location header." };
383
+ try {
384
+ currentUrl = new URL(location, currentUrl).toString();
385
+ } catch {
386
+ return { error: "Failed to fetch URL: the redirect has an invalid Location." };
387
+ }
388
+ continue;
389
+ }
390
+ if (response.status !== 200) return { error: statusError(response.status) };
391
+ const contentType = headerValue(response.headers, "content-type").toLowerCase();
392
+ const declaredPdf = contentType.includes("pdf");
393
+ const body = decodeBody(response.body, response.encoding);
394
+ if (!declaredPdf && !hasPdfMagic(body)) return null;
395
+ let pdfText: string;
396
+ try {
397
+ pdfText = await extractPdfText(body);
398
+ } catch {
399
+ return { error: "(PDF content could not be read as text)" };
400
+ }
401
+ if (!pdfText) return { error: "(PDF contains no extractable text)" };
402
+ return { text: pdfText };
403
+ }
404
+ return { error: "Failed to fetch URL: too many redirects." };
405
+ }