pi-unsloth-webtools 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +141 -75
- package/html-to-md.ts +19 -4
- package/index.ts +52 -70
- package/lightpanda.ts +276 -0
- package/package.json +13 -5
- package/scripts/install-lightpanda.sh +206 -0
- package/settings.ts +57 -11
- package/tls-fetch.ts +205 -0
- package/web-fetch.ts +72 -26
- package/config-ui.ts +0 -105
- package/config.ts +0 -207
- package/web-render.ts +0 -100
package/tls-fetch.ts
ADDED
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
import { MAX_SIGNAL_TIMEOUT_MS } from "./web-access.ts";
|
|
2
|
+
|
|
3
|
+
const MODULE_NAME = "wreq-js";
|
|
4
|
+
const DEFAULT_PROFILE = "chrome_145";
|
|
5
|
+
const DEFAULT_OS = "windows";
|
|
6
|
+
const TRANSPORT_CACHE_LIMIT = 8;
|
|
7
|
+
|
|
8
|
+
const HOP_HEADER_EXCLUSIONS = new Set(["host", "user-agent", "accept-encoding", "content-length"]);
|
|
9
|
+
const PDF_MAGIC = Buffer.from("%PDF-");
|
|
10
|
+
const MAGIC_HEAD_BYTES = 1024;
|
|
11
|
+
const SKIPPED_MAGIC_PREFIX_BYTES = new Set([0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x20]);
|
|
12
|
+
const BOM_PREFIXES = [
|
|
13
|
+
Buffer.from([0xff, 0xfe, 0x00, 0x00]),
|
|
14
|
+
Buffer.from([0x00, 0x00, 0xfe, 0xff]),
|
|
15
|
+
Buffer.from([0xff, 0xfe]),
|
|
16
|
+
Buffer.from([0xfe, 0xff]),
|
|
17
|
+
Buffer.from([0xef, 0xbb, 0xbf]),
|
|
18
|
+
];
|
|
19
|
+
|
|
20
|
+
function looksLikePdf(head: Buffer): boolean {
|
|
21
|
+
let start = 0;
|
|
22
|
+
while (start < head.length && SKIPPED_MAGIC_PREFIX_BYTES.has(head[start])) start++;
|
|
23
|
+
for (const bom of BOM_PREFIXES) {
|
|
24
|
+
if (head.subarray(start).length >= bom.length && head.subarray(start, start + bom.length).equals(bom)) {
|
|
25
|
+
start += bom.length;
|
|
26
|
+
while (start < head.length && SKIPPED_MAGIC_PREFIX_BYTES.has(head[start])) start++;
|
|
27
|
+
break;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
const magic = head.subarray(start);
|
|
31
|
+
return magic.length >= PDF_MAGIC.length && magic.subarray(0, PDF_MAGIC.length).equals(PDF_MAGIC);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
interface ImpersonationResponse {
|
|
35
|
+
status: number;
|
|
36
|
+
url: string;
|
|
37
|
+
headers: { get(name: string): string | null };
|
|
38
|
+
body: AsyncIterable<Uint8Array> | null;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
interface ImpersonationTransport {
|
|
42
|
+
close?(): Promise<void>;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
interface ImpersonationModule {
|
|
46
|
+
createTransport(options: Record<string, unknown>): Promise<ImpersonationTransport>;
|
|
47
|
+
fetch(input: string, init: Record<string, unknown>): Promise<ImpersonationResponse>;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export type ImpersonationLoader = () => Promise<ImpersonationModule | null>;
|
|
51
|
+
|
|
52
|
+
export interface TlsHopOptions {
|
|
53
|
+
url: URL;
|
|
54
|
+
pinnedIp: string;
|
|
55
|
+
family: number;
|
|
56
|
+
timeoutMs: number;
|
|
57
|
+
signal?: AbortSignal;
|
|
58
|
+
maxBytes: number;
|
|
59
|
+
maxPdfBytes?: number;
|
|
60
|
+
profile?: string;
|
|
61
|
+
extraHeaders?: Record<string, string>;
|
|
62
|
+
loader?: ImpersonationLoader;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export interface TlsHopResponse {
|
|
66
|
+
status: number;
|
|
67
|
+
headers: Record<string, string>;
|
|
68
|
+
body: Buffer;
|
|
69
|
+
truncated: boolean;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
let cachedModule: Promise<ImpersonationModule | null> | null = null;
|
|
73
|
+
|
|
74
|
+
async function loadImpersonationModule(): Promise<ImpersonationModule | null> {
|
|
75
|
+
cachedModule ??= (async () => {
|
|
76
|
+
try {
|
|
77
|
+
return (await import(MODULE_NAME)) as unknown as ImpersonationModule;
|
|
78
|
+
} catch {
|
|
79
|
+
return null;
|
|
80
|
+
}
|
|
81
|
+
})();
|
|
82
|
+
return cachedModule;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
const transports = new Map<string, Promise<ImpersonationTransport>>();
|
|
86
|
+
|
|
87
|
+
function transportKey(options: TlsHopOptions): string {
|
|
88
|
+
return `${options.profile ?? DEFAULT_PROFILE}|${options.url.hostname}|${options.pinnedIp}|${options.family}`;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function evictTransports(): void {
|
|
92
|
+
while (transports.size > TRANSPORT_CACHE_LIMIT) {
|
|
93
|
+
const oldestKey = transports.keys().next().value as string | undefined;
|
|
94
|
+
if (oldestKey === undefined) return;
|
|
95
|
+
const oldest = transports.get(oldestKey);
|
|
96
|
+
transports.delete(oldestKey);
|
|
97
|
+
void oldest
|
|
98
|
+
?.then((transport) => transport.close?.())
|
|
99
|
+
.catch(() => {});
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
async function transportFor(
|
|
104
|
+
module: ImpersonationModule,
|
|
105
|
+
options: TlsHopOptions,
|
|
106
|
+
): Promise<ImpersonationTransport> {
|
|
107
|
+
const key = transportKey(options);
|
|
108
|
+
const existing = transports.get(key);
|
|
109
|
+
if (existing) return existing;
|
|
110
|
+
const created = module
|
|
111
|
+
.createTransport({
|
|
112
|
+
resolve: { [options.url.hostname]: options.pinnedIp },
|
|
113
|
+
browser: options.profile ?? DEFAULT_PROFILE,
|
|
114
|
+
os: DEFAULT_OS,
|
|
115
|
+
})
|
|
116
|
+
.catch((error: unknown) => {
|
|
117
|
+
transports.delete(key);
|
|
118
|
+
throw error;
|
|
119
|
+
});
|
|
120
|
+
transports.set(key, created);
|
|
121
|
+
evictTransports();
|
|
122
|
+
return created;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
async function readCapped(
|
|
126
|
+
body: AsyncIterable<Uint8Array> | null,
|
|
127
|
+
maxBytes: number,
|
|
128
|
+
maxPdfBytes: number,
|
|
129
|
+
): Promise<{ buffer: Buffer; truncated: boolean }> {
|
|
130
|
+
if (!body) return { buffer: Buffer.alloc(0), truncated: false };
|
|
131
|
+
const chunks: Buffer[] = [];
|
|
132
|
+
let total = 0;
|
|
133
|
+
let truncated = false;
|
|
134
|
+
let head = Buffer.alloc(0);
|
|
135
|
+
let limit = maxBytes;
|
|
136
|
+
for await (const chunk of body) {
|
|
137
|
+
const buffer = Buffer.from(chunk);
|
|
138
|
+
if (head.length < MAGIC_HEAD_BYTES) {
|
|
139
|
+
head = Buffer.concat([head, buffer.subarray(0, MAGIC_HEAD_BYTES - head.length)]);
|
|
140
|
+
if (limit === maxBytes && looksLikePdf(head)) limit = maxPdfBytes;
|
|
141
|
+
}
|
|
142
|
+
if (total + buffer.length > limit) {
|
|
143
|
+
const take = buffer.subarray(0, limit - total);
|
|
144
|
+
chunks.push(take);
|
|
145
|
+
total += take.length;
|
|
146
|
+
truncated = true;
|
|
147
|
+
break;
|
|
148
|
+
}
|
|
149
|
+
chunks.push(buffer);
|
|
150
|
+
total += buffer.length;
|
|
151
|
+
}
|
|
152
|
+
return { buffer: Buffer.concat(chunks), truncated };
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
function failureMessage(error: unknown, caller: AbortSignal | undefined, attempt: AbortSignal): string {
|
|
156
|
+
if (caller?.aborted) return "cancelled";
|
|
157
|
+
if (attempt.aborted) return "timed out";
|
|
158
|
+
if (error instanceof Error && error.name === "TimeoutError") return "timed out";
|
|
159
|
+
if (error instanceof Error && error.name === "AbortError") return "cancelled";
|
|
160
|
+
return error instanceof Error ? error.message : String(error);
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function hopHeaders(extraHeaders: Record<string, string> | undefined): Record<string, string> {
|
|
164
|
+
const headers: Record<string, string> = {};
|
|
165
|
+
for (const [key, value] of Object.entries(extraHeaders ?? {})) {
|
|
166
|
+
if (HOP_HEADER_EXCLUSIONS.has(key.toLowerCase())) continue;
|
|
167
|
+
headers[key] = value;
|
|
168
|
+
}
|
|
169
|
+
return headers;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
export async function impersonatedRequest(options: TlsHopOptions): Promise<TlsHopResponse | null> {
|
|
173
|
+
const load = options.loader ?? loadImpersonationModule;
|
|
174
|
+
const module = await load();
|
|
175
|
+
if (!module) return null;
|
|
176
|
+
const transport = await transportFor(module, options);
|
|
177
|
+
const timeoutMs = Math.min(MAX_SIGNAL_TIMEOUT_MS, Math.max(1, Math.floor(options.timeoutMs)));
|
|
178
|
+
const timeoutSignal = AbortSignal.timeout(timeoutMs);
|
|
179
|
+
const signal = options.signal ? AbortSignal.any([options.signal, timeoutSignal]) : timeoutSignal;
|
|
180
|
+
let response: ImpersonationResponse;
|
|
181
|
+
try {
|
|
182
|
+
response = await module.fetch(options.url.toString(), {
|
|
183
|
+
transport,
|
|
184
|
+
redirect: "manual",
|
|
185
|
+
signal,
|
|
186
|
+
headers: hopHeaders(options.extraHeaders),
|
|
187
|
+
});
|
|
188
|
+
} catch (error) {
|
|
189
|
+
throw new Error(failureMessage(error, options.signal, signal));
|
|
190
|
+
}
|
|
191
|
+
if (response.status < 200) return null;
|
|
192
|
+
const { buffer, truncated } = await readCapped(
|
|
193
|
+
response.body,
|
|
194
|
+
options.maxBytes,
|
|
195
|
+
options.maxPdfBytes ?? options.maxBytes,
|
|
196
|
+
);
|
|
197
|
+
const headers: Record<string, string> = {};
|
|
198
|
+
const contentType = response.headers.get("content-type");
|
|
199
|
+
if (contentType) headers["content-type"] = contentType;
|
|
200
|
+
if (response.status >= 300 && response.status < 400) {
|
|
201
|
+
const location = response.headers.get("location");
|
|
202
|
+
if (location) headers.location = location;
|
|
203
|
+
}
|
|
204
|
+
return { status: response.status, headers, body: buffer, truncated };
|
|
205
|
+
}
|
package/web-fetch.ts
CHANGED
|
@@ -23,6 +23,7 @@ import {
|
|
|
23
23
|
stripIpv6Brackets,
|
|
24
24
|
type WebsitePolicy,
|
|
25
25
|
} from "./web-access.ts";
|
|
26
|
+
import { impersonatedRequest, type TlsHopOptions, type TlsHopResponse } from "./tls-fetch.ts";
|
|
26
27
|
import { collapseWhitespace, decodeHtmlEntities, feedHtml, htmlToMarkdown, visibleChars } from "./html-to-md.ts";
|
|
27
28
|
import type { AttrDict } from "./html-to-md.ts";
|
|
28
29
|
import { INVALID_CHARREFS } from "./entities.ts";
|
|
@@ -157,8 +158,8 @@ export class FetchTimeoutError extends Error {
|
|
|
157
158
|
}
|
|
158
159
|
const FETCH_CANCELLED_MESSAGE = "Failed to fetch URL: cancelled.";
|
|
159
160
|
const FETCH_TIMEOUT_MESSAGE = "Failed to fetch URL: timed out.";
|
|
160
|
-
const TRUNCATED_BODY_SUFFIX = "... (page truncated at the download limit)";
|
|
161
|
-
const TRUNCATED_BODY_NOTICE = "\n\n" + TRUNCATED_BODY_SUFFIX;
|
|
161
|
+
export const TRUNCATED_BODY_SUFFIX = "... (page truncated at the download limit)";
|
|
162
|
+
export const TRUNCATED_BODY_NOTICE = "\n\n" + TRUNCATED_BODY_SUFFIX;
|
|
162
163
|
|
|
163
164
|
function fetchErrorMessage(err: unknown): string {
|
|
164
165
|
if (err instanceof FetchCancelledError) return FETCH_CANCELLED_MESSAGE;
|
|
@@ -188,6 +189,7 @@ export interface FetchPageOptions {
|
|
|
188
189
|
maxBytes?: number;
|
|
189
190
|
maxPdfBytes?: number;
|
|
190
191
|
allowPrivateAddresses?: boolean;
|
|
192
|
+
transport?: FetchTransport;
|
|
191
193
|
allowLocalFiles?: boolean;
|
|
192
194
|
seams?: FetchSeams;
|
|
193
195
|
rawFetch?: (url: string, options: RawFetchOptions) => Promise<RawFetchResult>;
|
|
@@ -208,9 +210,13 @@ export interface ResolvedHost {
|
|
|
208
210
|
alternates?: { ip: string; family: number }[];
|
|
209
211
|
}
|
|
210
212
|
|
|
213
|
+
export type FetchTransport = "tls-first" | "direct-first" | "off";
|
|
214
|
+
|
|
215
|
+
type TransportKind = "direct" | "tls";
|
|
211
216
|
export interface FetchSeams {
|
|
212
217
|
resolve?: (hostname: string, signal?: AbortSignal, allowPrivateAddresses?: boolean) => Promise<ResolvedHost>;
|
|
213
218
|
request?: (opts: HopOptions) => Promise<HopResponse>;
|
|
219
|
+
impersonate?: (options: TlsHopOptions) => Promise<TlsHopResponse | null>;
|
|
214
220
|
}
|
|
215
221
|
|
|
216
222
|
export interface HopOptions {
|
|
@@ -236,6 +242,7 @@ export interface RawFetchOptions {
|
|
|
236
242
|
maxBytes?: number;
|
|
237
243
|
maxPdfBytes?: number;
|
|
238
244
|
allowPrivateAddresses?: boolean;
|
|
245
|
+
transport?: FetchTransport;
|
|
239
246
|
seams?: FetchSeams;
|
|
240
247
|
}
|
|
241
248
|
|
|
@@ -903,6 +910,10 @@ export async function fetchUrlRaw(
|
|
|
903
910
|
const resolveHost = seams.resolve ?? resolveAndValidateHost;
|
|
904
911
|
const allowPrivateAddresses = options.allowPrivateAddresses ?? true;
|
|
905
912
|
const performRequest = seams.request ?? requestHop;
|
|
913
|
+
const transportMode = options.transport ?? "tls-first";
|
|
914
|
+
const performImpersonation =
|
|
915
|
+
transportMode === "off" ? null : (seams.impersonate ?? (seams.request ? null : impersonatedRequest));
|
|
916
|
+
const directFirst = transportMode === "direct-first" || performImpersonation === null;
|
|
906
917
|
const resolveWithBudget = async (hostname: string): Promise<ResolvedHost> => {
|
|
907
918
|
const abortController = new AbortController();
|
|
908
919
|
const waitMs = clampedRemainingMs(deadline, now);
|
|
@@ -937,6 +948,7 @@ export async function fetchUrlRaw(
|
|
|
937
948
|
let pinnedFamily = resolved.family;
|
|
938
949
|
let alternates: { ip: string; family: number }[] = resolved.alternates ?? [];
|
|
939
950
|
let alternateIndex = 0;
|
|
951
|
+
const triedTransports = new Set<string>();
|
|
940
952
|
const userAgent = randomUserAgent();
|
|
941
953
|
for (let hop = 0; hop < MAX_REQUESTS; hop++) {
|
|
942
954
|
const budgetResult = checkBudget();
|
|
@@ -960,34 +972,67 @@ export async function fetchUrlRaw(
|
|
|
960
972
|
}
|
|
961
973
|
headers["Host"] = hostHeader;
|
|
962
974
|
const inactivity = remainingMs(deadline, now);
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
975
|
+
const order: TransportKind[] = socksProxyForUrl(parsed) !== null
|
|
976
|
+
? ["direct"]
|
|
977
|
+
: directFirst
|
|
978
|
+
? ["direct", "tls"]
|
|
979
|
+
: ["tls", "direct"];
|
|
980
|
+
let response: HopResponse | null = null;
|
|
981
|
+
let failure: unknown = null;
|
|
982
|
+
for (const kind of order) {
|
|
983
|
+
const key = `${kind}|${currentUrl}|${pinnedIp}`;
|
|
984
|
+
if (triedTransports.has(key)) continue;
|
|
985
|
+
triedTransports.add(key);
|
|
986
|
+
let attempt: HopResponse | null;
|
|
987
|
+
try {
|
|
988
|
+
if (kind === "tls") {
|
|
989
|
+
const impersonate = performImpersonation;
|
|
990
|
+
if (impersonate === null) continue;
|
|
991
|
+
attempt = await impersonate({
|
|
992
|
+
url: parsed,
|
|
993
|
+
pinnedIp,
|
|
994
|
+
family: pinnedFamily,
|
|
995
|
+
timeoutMs: remainingMs(deadline, now),
|
|
996
|
+
signal,
|
|
997
|
+
maxBytes,
|
|
998
|
+
maxPdfBytes,
|
|
999
|
+
extraHeaders: options.extraHeaders,
|
|
1000
|
+
});
|
|
1001
|
+
} else {
|
|
1002
|
+
attempt = await performRequest({
|
|
1003
|
+
url: parsed,
|
|
1004
|
+
pinnedIp,
|
|
1005
|
+
family: pinnedFamily,
|
|
1006
|
+
headers,
|
|
1007
|
+
maxBytes,
|
|
1008
|
+
maxPdfBytes,
|
|
1009
|
+
inactivityMs: inactivity,
|
|
1010
|
+
deadlineMs: deadline,
|
|
1011
|
+
nowMs: now,
|
|
1012
|
+
signal,
|
|
1013
|
+
});
|
|
1014
|
+
}
|
|
1015
|
+
} catch (err) {
|
|
1016
|
+
const message = err instanceof Error ? err.message : "";
|
|
1017
|
+
if (message === "cancelled" || message === "timed out") return emptyResult(fetchErrorMessage(err));
|
|
1018
|
+
if (err instanceof FetchCancelledError || err instanceof FetchTimeoutError) {
|
|
1019
|
+
return emptyResult(fetchErrorMessage(err));
|
|
1020
|
+
}
|
|
1021
|
+
failure = err;
|
|
1022
|
+
continue;
|
|
1023
|
+
}
|
|
1024
|
+
if (attempt === null) continue;
|
|
1025
|
+
if (response === null || (attempt.status < 400 && response.status >= 400)) response = attempt;
|
|
1026
|
+
if (attempt.status < 400) break;
|
|
1027
|
+
}
|
|
1028
|
+
if (response === null) {
|
|
1029
|
+
if (alternateIndex < alternates.length) {
|
|
985
1030
|
const next = alternates[alternateIndex++];
|
|
986
1031
|
pinnedIp = next.ip;
|
|
987
1032
|
pinnedFamily = next.family;
|
|
988
1033
|
continue;
|
|
989
1034
|
}
|
|
990
|
-
return emptyResult(fetchErrorMessage(
|
|
1035
|
+
return emptyResult(fetchErrorMessage(failure ?? new Error("no transport could reach the target")));
|
|
991
1036
|
}
|
|
992
1037
|
|
|
993
1038
|
if (response.status >= 300 && response.status < 400) {
|
|
@@ -1256,7 +1301,7 @@ function extractPageMeta(html: string): PageMeta {
|
|
|
1256
1301
|
return meta;
|
|
1257
1302
|
}
|
|
1258
1303
|
|
|
1259
|
-
function pagePrefixedMarkdown(html: string): string {
|
|
1304
|
+
export function pagePrefixedMarkdown(html: string): string {
|
|
1260
1305
|
const meta = extractPageMeta(html);
|
|
1261
1306
|
const lines: string[] = [];
|
|
1262
1307
|
if (meta.title) lines.push(`Title: ${meta.title}`);
|
|
@@ -1446,6 +1491,7 @@ export async function fetchPageOutcome(
|
|
|
1446
1491
|
maxBytes: options.maxBytes,
|
|
1447
1492
|
maxPdfBytes: options.maxPdfBytes,
|
|
1448
1493
|
seams: options.seams,
|
|
1494
|
+
transport: options.transport,
|
|
1449
1495
|
};
|
|
1450
1496
|
const useCache = !options.seams && !options.rawFetch;
|
|
1451
1497
|
const githubRaw = githubRawContentUrl(url);
|
package/config-ui.ts
DELETED
|
@@ -1,105 +0,0 @@
|
|
|
1
|
-
import type { Theme } from "@earendil-works/pi-coding-agent";
|
|
2
|
-
import { Key, matchesKey, visibleWidth } from "@earendil-works/pi-tui";
|
|
3
|
-
import { readConfig, type Config } from "./config.ts";
|
|
4
|
-
|
|
5
|
-
export type ConfigToggleKey = "webRenderEnabled";
|
|
6
|
-
|
|
7
|
-
export interface ConfigRow {
|
|
8
|
-
key: ConfigToggleKey;
|
|
9
|
-
label: string;
|
|
10
|
-
hint: string;
|
|
11
|
-
enabled: boolean;
|
|
12
|
-
}
|
|
13
|
-
|
|
14
|
-
export function configRows(config: Config): ConfigRow[] {
|
|
15
|
-
return [
|
|
16
|
-
{
|
|
17
|
-
key: "webRenderEnabled",
|
|
18
|
-
label: "JavaScript rendering",
|
|
19
|
-
hint: "Jina Reader fallback",
|
|
20
|
-
enabled: config.webRenderEnabled !== false,
|
|
21
|
-
},
|
|
22
|
-
];
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
function padRow(theme: Theme, innerWidth: number, content: string): string {
|
|
26
|
-
const padded = content + " ".repeat(Math.max(0, innerWidth - visibleWidth(content)));
|
|
27
|
-
return theme.fg("border", "│") + padded + theme.fg("border", "│");
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
export class WebToolsConfigOverlay {
|
|
31
|
-
private rows: ConfigRow[] = [];
|
|
32
|
-
private selected = 0;
|
|
33
|
-
|
|
34
|
-
constructor(
|
|
35
|
-
private readonly opts: {
|
|
36
|
-
tui: { requestRender(force?: boolean): void };
|
|
37
|
-
theme: Theme;
|
|
38
|
-
done: () => void;
|
|
39
|
-
onToggle: (key: ConfigToggleKey) => Promise<void>;
|
|
40
|
-
},
|
|
41
|
-
) {}
|
|
42
|
-
|
|
43
|
-
async load(): Promise<void> {
|
|
44
|
-
this.rows = configRows(await readConfig());
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
private runToggle(row: ConfigRow): void {
|
|
48
|
-
this.opts.tui.requestRender(true);
|
|
49
|
-
void this.opts
|
|
50
|
-
.onToggle(row.key)
|
|
51
|
-
.then(async () => {
|
|
52
|
-
this.rows = configRows(await readConfig());
|
|
53
|
-
this.opts.tui.requestRender(true);
|
|
54
|
-
})
|
|
55
|
-
.catch((error: unknown) => {
|
|
56
|
-
console.error("Failed to toggle web tools setting:", error);
|
|
57
|
-
});
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
private toggleSelected(): void {
|
|
61
|
-
const row = this.rows[this.selected];
|
|
62
|
-
if (!row) return;
|
|
63
|
-
row.enabled = !row.enabled;
|
|
64
|
-
this.runToggle(row);
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
handleInput(data: string): void {
|
|
68
|
-
if (matchesKey(data, Key.up) || data === "k") {
|
|
69
|
-
this.selected = (this.selected + this.rows.length - 1) % this.rows.length;
|
|
70
|
-
return;
|
|
71
|
-
}
|
|
72
|
-
if (matchesKey(data, Key.down) || data === "j") {
|
|
73
|
-
this.selected = (this.selected + 1) % this.rows.length;
|
|
74
|
-
return;
|
|
75
|
-
}
|
|
76
|
-
if (matchesKey(data, Key.space) || matchesKey(data, Key.enter) || data === " " || data === "\r" || data === "\n") {
|
|
77
|
-
this.toggleSelected();
|
|
78
|
-
return;
|
|
79
|
-
}
|
|
80
|
-
if (matchesKey(data, Key.escape) || data === "q") {
|
|
81
|
-
this.opts.done();
|
|
82
|
-
}
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
invalidate(): void {}
|
|
86
|
-
|
|
87
|
-
render(width: number): string[] {
|
|
88
|
-
const theme = this.opts.theme;
|
|
89
|
-
const innerWidth = width - 2;
|
|
90
|
-
const lines: string[] = [];
|
|
91
|
-
lines.push(theme.fg("border", `╭${"─".repeat(innerWidth)}╮`));
|
|
92
|
-
lines.push(padRow(theme, innerWidth, ` ${theme.fg("accent", theme.bold("Web Tools Config"))}`));
|
|
93
|
-
lines.push(theme.fg("border", `├${"─".repeat(innerWidth)}┤`));
|
|
94
|
-
this.rows.forEach((row, index) => {
|
|
95
|
-
const cursor = index === this.selected ? theme.fg("accent", "> ") : " ";
|
|
96
|
-
const box = row.enabled ? theme.fg("success", "[x]") : theme.fg("dim", "[ ]");
|
|
97
|
-
const label = index === this.selected ? theme.fg("accent", theme.bold(row.label)) : row.label;
|
|
98
|
-
lines.push(padRow(theme, innerWidth, `${cursor}${box} ${label} ${theme.fg("dim", `— ${row.hint}`)}`));
|
|
99
|
-
});
|
|
100
|
-
lines.push(theme.fg("border", `├${"─".repeat(innerWidth)}┤`));
|
|
101
|
-
lines.push(padRow(theme, innerWidth, theme.fg("dim", " ↑↓ navigate · space toggle · q close")));
|
|
102
|
-
lines.push(theme.fg("border", `╰${"─".repeat(innerWidth)}╯`));
|
|
103
|
-
return lines;
|
|
104
|
-
}
|
|
105
|
-
}
|
package/config.ts
DELETED
|
@@ -1,207 +0,0 @@
|
|
|
1
|
-
import { randomUUID } from "node:crypto";
|
|
2
|
-
import { mkdir, open, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
|
|
3
|
-
import { homedir } from "node:os";
|
|
4
|
-
import { dirname, join } from "node:path";
|
|
5
|
-
|
|
6
|
-
const CONFIG_DIR_NAME = "pi-unsloth-webtools";
|
|
7
|
-
const TEMP_PREFIX = ".tmp-";
|
|
8
|
-
const CONFIG_LOCK_DELAY_MS = 25;
|
|
9
|
-
const CONFIG_LOCK_STALE_MS = 5000;
|
|
10
|
-
const CONFIG_LOCK_RETRIES = Math.ceil(CONFIG_LOCK_STALE_MS / CONFIG_LOCK_DELAY_MS) * 2;
|
|
11
|
-
|
|
12
|
-
export interface Config {
|
|
13
|
-
webRenderEnabled: boolean;
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
const DEFAULT_CONFIG: Config = {
|
|
17
|
-
webRenderEnabled: true,
|
|
18
|
-
};
|
|
19
|
-
|
|
20
|
-
function homeBase(): string {
|
|
21
|
-
const envHome = process.env.HOME;
|
|
22
|
-
return envHome && envHome.length > 0 ? envHome : homedir();
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
function configBase(): string {
|
|
26
|
-
if (process.platform !== "win32") {
|
|
27
|
-
const xdg = process.env.XDG_CONFIG_HOME;
|
|
28
|
-
if (xdg && xdg.length > 0) return xdg;
|
|
29
|
-
}
|
|
30
|
-
return join(homeBase(), ".config");
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
export function configDir(): string {
|
|
34
|
-
return join(configBase(), CONFIG_DIR_NAME);
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
export function configPath(): string {
|
|
38
|
-
return join(configDir(), "config.json");
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
42
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
function errCode(error: unknown): string | undefined {
|
|
46
|
-
if (typeof error !== "object" || error === null) return undefined;
|
|
47
|
-
const code = (error as { code?: unknown }).code;
|
|
48
|
-
return typeof code === "string" ? code : undefined;
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
function parseConfig(content: string): Config {
|
|
52
|
-
const parsed = JSON.parse(content) as unknown;
|
|
53
|
-
if (!isRecord(parsed) || (parsed.webRenderEnabled !== undefined && typeof parsed.webRenderEnabled !== "boolean")) {
|
|
54
|
-
throw new Error("config.json must be an object with a boolean webRenderEnabled field");
|
|
55
|
-
}
|
|
56
|
-
return {
|
|
57
|
-
webRenderEnabled:
|
|
58
|
-
typeof parsed.webRenderEnabled === "boolean" ? parsed.webRenderEnabled : DEFAULT_CONFIG.webRenderEnabled,
|
|
59
|
-
};
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
async function loadConfigFile(): Promise<{ config: Config; corrupted: boolean }> {
|
|
63
|
-
let content: string;
|
|
64
|
-
try {
|
|
65
|
-
content = await readFile(configPath(), "utf-8");
|
|
66
|
-
} catch (error: unknown) {
|
|
67
|
-
if (errCode(error) === "ENOENT") return { config: { ...DEFAULT_CONFIG }, corrupted: false };
|
|
68
|
-
console.error("Config file unreadable, using defaults:", error);
|
|
69
|
-
return { config: { ...DEFAULT_CONFIG }, corrupted: false };
|
|
70
|
-
}
|
|
71
|
-
try {
|
|
72
|
-
return { config: parseConfig(content), corrupted: false };
|
|
73
|
-
} catch (error: unknown) {
|
|
74
|
-
try {
|
|
75
|
-
const badPath = configPath();
|
|
76
|
-
await rename(badPath, `${badPath}.corrupt-${Date.now()}-${process.pid}-${Math.random().toString(36).slice(2)}`);
|
|
77
|
-
} catch {}
|
|
78
|
-
console.error("Config file corrupted, quarantined, using defaults:", error);
|
|
79
|
-
return { config: { ...DEFAULT_CONFIG }, corrupted: true };
|
|
80
|
-
}
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
export async function readConfig(): Promise<Config> {
|
|
84
|
-
return (await loadConfigFile()).config;
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
export async function readConfigWithStatus(): Promise<{ config: Config; corrupted: boolean }> {
|
|
88
|
-
return loadConfigFile();
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
async function syncDir(dir: string): Promise<void> {
|
|
92
|
-
if (process.platform === "win32") return;
|
|
93
|
-
try {
|
|
94
|
-
const handle = await open(dir, "r");
|
|
95
|
-
try {
|
|
96
|
-
await handle.sync();
|
|
97
|
-
} finally {
|
|
98
|
-
await handle.close();
|
|
99
|
-
}
|
|
100
|
-
} catch {}
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
export async function writeConfig(config: Config): Promise<void> {
|
|
104
|
-
const path = configPath();
|
|
105
|
-
const dir = dirname(path);
|
|
106
|
-
await mkdir(dir, { recursive: true, mode: 0o700 });
|
|
107
|
-
const tempPath = join(dir, `${TEMP_PREFIX}${randomUUID()}`);
|
|
108
|
-
const content = JSON.stringify(config, null, 2);
|
|
109
|
-
await writeFile(tempPath, content, { mode: 0o600 });
|
|
110
|
-
try {
|
|
111
|
-
await rename(tempPath, path);
|
|
112
|
-
await syncDir(dir);
|
|
113
|
-
} catch (error: unknown) {
|
|
114
|
-
if (process.platform === "win32" && errCode(error) === "EPERM") {
|
|
115
|
-
try {
|
|
116
|
-
await writeFile(path, content, "utf-8");
|
|
117
|
-
return;
|
|
118
|
-
} finally {
|
|
119
|
-
try {
|
|
120
|
-
await rm(tempPath, { force: true });
|
|
121
|
-
} catch {}
|
|
122
|
-
}
|
|
123
|
-
}
|
|
124
|
-
try {
|
|
125
|
-
await rm(tempPath, { force: true });
|
|
126
|
-
} catch {}
|
|
127
|
-
throw error;
|
|
128
|
-
}
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
interface ConfigLock {
|
|
132
|
-
path: string;
|
|
133
|
-
dev: number;
|
|
134
|
-
ino: number;
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
async function acquireConfigLock(lockPath: string): Promise<ConfigLock> {
|
|
138
|
-
try {
|
|
139
|
-
await mkdir(dirname(lockPath), { recursive: true, mode: 0o700 });
|
|
140
|
-
} catch {}
|
|
141
|
-
for (let attempt = 0; attempt < CONFIG_LOCK_RETRIES; attempt++) {
|
|
142
|
-
try {
|
|
143
|
-
await mkdir(lockPath, { mode: 0o700 });
|
|
144
|
-
} catch (error: unknown) {
|
|
145
|
-
if (errCode(error) === "ENOENT") {
|
|
146
|
-
try {
|
|
147
|
-
await mkdir(dirname(lockPath), { recursive: true, mode: 0o700 });
|
|
148
|
-
} catch {}
|
|
149
|
-
continue;
|
|
150
|
-
}
|
|
151
|
-
if (errCode(error) !== "EEXIST") throw error;
|
|
152
|
-
try {
|
|
153
|
-
const lockStats = await stat(lockPath);
|
|
154
|
-
if (Date.now() - lockStats.mtimeMs > CONFIG_LOCK_STALE_MS) {
|
|
155
|
-
await rm(lockPath, { recursive: true, force: true });
|
|
156
|
-
continue;
|
|
157
|
-
}
|
|
158
|
-
} catch {}
|
|
159
|
-
await new Promise<void>((resolve) => setTimeout(resolve, CONFIG_LOCK_DELAY_MS));
|
|
160
|
-
continue;
|
|
161
|
-
}
|
|
162
|
-
try {
|
|
163
|
-
const lockStats = await stat(lockPath);
|
|
164
|
-
return { path: lockPath, dev: lockStats.dev, ino: lockStats.ino };
|
|
165
|
-
} catch (error: unknown) {
|
|
166
|
-
if (errCode(error) !== "ENOENT") {
|
|
167
|
-
try {
|
|
168
|
-
await rm(lockPath, { recursive: true, force: true });
|
|
169
|
-
} catch {}
|
|
170
|
-
}
|
|
171
|
-
continue;
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
throw new Error(`[E_ACCESS] Could not acquire config lock: ${lockPath}`);
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
async function releaseConfigLock(lock: ConfigLock): Promise<void> {
|
|
178
|
-
try {
|
|
179
|
-
const lockStats = await stat(lock.path);
|
|
180
|
-
if (lockStats.dev !== lock.dev || lockStats.ino !== lock.ino) return;
|
|
181
|
-
} catch {
|
|
182
|
-
return;
|
|
183
|
-
}
|
|
184
|
-
try {
|
|
185
|
-
await rm(lock.path, { recursive: true, force: true });
|
|
186
|
-
} catch {}
|
|
187
|
-
}
|
|
188
|
-
|
|
189
|
-
export async function updateConfig(mut: (config: Config) => void): Promise<Config> {
|
|
190
|
-
const path = configPath();
|
|
191
|
-
const lock = await acquireConfigLock(`${path}.lock`);
|
|
192
|
-
try {
|
|
193
|
-
const config = await readConfig();
|
|
194
|
-
mut(config);
|
|
195
|
-
await writeConfig(config);
|
|
196
|
-
return config;
|
|
197
|
-
} finally {
|
|
198
|
-
await releaseConfigLock(lock);
|
|
199
|
-
}
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
export async function toggleWebRender(): Promise<boolean> {
|
|
203
|
-
const config = await updateConfig((current) => {
|
|
204
|
-
current.webRenderEnabled = !(current.webRenderEnabled === true);
|
|
205
|
-
});
|
|
206
|
-
return config.webRenderEnabled === true;
|
|
207
|
-
}
|