pi-hashline-edit-pro 2.7.2 → 2.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -1
- package/index.ts +9 -6
- package/package.json +10 -10
- package/prompts/grep-guidelines.md +3 -7
- package/prompts/grep-snippet.md +1 -1
- package/prompts/grep.md +1 -1
- package/prompts/insert-guidelines.md +3 -7
- package/prompts/insert-snippet.md +1 -1
- package/prompts/insert.md +1 -1
- package/prompts/read-guidelines.md +1 -1
- package/prompts/read.md +1 -1
- package/prompts/replace-guidelines.md +5 -9
- package/prompts/replace-snippet.md +1 -1
- package/prompts/replace.md +1 -1
- package/prompts/undo-last-change-guidelines.md +2 -4
- package/prompts/undo-last-change-snippet.md +1 -1
- package/prompts/undo-last-change.md +1 -1
- package/src/commit.ts +2 -1
- package/src/edit-common.ts +45 -0
- package/src/file-kind.ts +14 -8
- package/src/file-reader.ts +34 -13
- package/src/fs-write.ts +60 -3
- package/src/grep.ts +119 -32
- package/src/hash-store/cache.ts +18 -0
- package/src/hash-store/retry.ts +48 -0
- package/src/hash-store/validation.ts +62 -0
- package/src/hash-store.ts +68 -133
- package/src/hashline/hash.ts +11 -9
- package/src/hashline/parse.ts +15 -1
- package/src/hashline/resolve.ts +3 -2
- package/src/insert.ts +11 -32
- package/src/normalize.ts +27 -0
- package/src/payload-contract.ts +102 -0
- package/src/read.ts +15 -1
- package/src/replace-diff.ts +28 -45
- package/src/replace-render.ts +18 -40
- package/src/replace-response.ts +1 -0
- package/src/replace-undo.ts +28 -13
- package/src/replace.ts +49 -128
- package/src/served.ts +26 -5
- package/src/utils.ts +58 -0
- package/src/validation.ts +2 -2
- package/src/write-hook.ts +59 -0
- package/src/replace-normalize.ts +0 -13
package/src/grep.ts
CHANGED
|
@@ -3,13 +3,12 @@ import { formatSize, DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES, type TruncationResult
|
|
|
3
3
|
import { Type } from "typebox";
|
|
4
4
|
import { readdir, stat } from "fs/promises";
|
|
5
5
|
import { dirname, join, relative } from "path";
|
|
6
|
-
import {
|
|
7
|
-
import { readNormFile } from "./file-reader";
|
|
6
|
+
import { tryReadNormFile } from "./file-reader";
|
|
8
7
|
import { MAX_HASH_LINES, fmtRow, HASH_LEN, HASH_SEP } from "./hashline";
|
|
9
8
|
import { MAX_GREP_LINE_BYTES } from "./constants";
|
|
10
9
|
import { toCwd } from "./paths";
|
|
11
10
|
import { loadP, loadGuide } from "./prompts";
|
|
12
|
-
import { normReq } from "./
|
|
11
|
+
import { normReq } from "./payload-contract";
|
|
13
12
|
import { recordServedSafe } from "./served";
|
|
14
13
|
import { abortIf, errCode, isRec, makePrepareArguments, rejectUnknownFields, truncateToBytes, visLines } from "./utils";
|
|
15
14
|
|
|
@@ -17,6 +16,10 @@ const GREP_KS = new Set(["pattern", "path", "glob", "context", "ignoreCase", "li
|
|
|
17
16
|
const SKIP_DIRS = new Set(["node_modules", ".git", ".tmp", "coverage"]);
|
|
18
17
|
const MAX_SCAN_FILES = 4000;
|
|
19
18
|
|
|
19
|
+
function cmp(a: string, b: string): number {
|
|
20
|
+
return a < b ? -1 : a > b ? 1 : 0;
|
|
21
|
+
}
|
|
22
|
+
|
|
20
23
|
export interface GrepReq {
|
|
21
24
|
pattern: string;
|
|
22
25
|
path?: string;
|
|
@@ -44,6 +47,7 @@ export function assertGrepReq(request: unknown): asserts request is GrepReq {
|
|
|
44
47
|
}
|
|
45
48
|
|
|
46
49
|
function buildRegex(pattern: string, literal: boolean, ignoreCase: boolean): RegExp {
|
|
50
|
+
if (!literal) assertSafeRegex(pattern);
|
|
47
51
|
const source = literal ? pattern.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") : pattern;
|
|
48
52
|
try {
|
|
49
53
|
return new RegExp(source, ignoreCase ? "ui" : "u");
|
|
@@ -52,7 +56,97 @@ function buildRegex(pattern: string, literal: boolean, ignoreCase: boolean): Reg
|
|
|
52
56
|
}
|
|
53
57
|
}
|
|
54
58
|
|
|
59
|
+
interface RegexGroupRisk {
|
|
60
|
+
hasQuantifier: boolean;
|
|
61
|
+
hasAlternation: boolean;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function unsafeRegex(pattern: string): never {
|
|
65
|
+
throw new Error(
|
|
66
|
+
`[E_UNSAFE_REGEX] Refusing potentially exponential regex: ${pattern}. Use literal: true or simplify the expression.`,
|
|
67
|
+
);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
function assertSafeRegex(pattern: string): void {
|
|
71
|
+
if (pattern.length > 4096) unsafeRegex(pattern);
|
|
72
|
+
|
|
73
|
+
const groups: RegexGroupRisk[] = [];
|
|
74
|
+
let inClass = false;
|
|
75
|
+
let escaped = false;
|
|
76
|
+
let variableQuantifiers = 0;
|
|
77
|
+
let lastAtom: { groupRisky: boolean; quantified: boolean } | undefined;
|
|
78
|
+
|
|
79
|
+
for (let i = 0; i < pattern.length; i++) {
|
|
80
|
+
const ch = pattern[i]!;
|
|
81
|
+
if (escaped) {
|
|
82
|
+
if (!inClass && (/[1-9]/.test(ch) || (ch === "k" && pattern[i + 1] === "<"))) {
|
|
83
|
+
unsafeRegex(pattern);
|
|
84
|
+
}
|
|
85
|
+
escaped = false;
|
|
86
|
+
lastAtom = { groupRisky: false, quantified: false };
|
|
87
|
+
continue;
|
|
88
|
+
}
|
|
89
|
+
if (ch === "\\") {
|
|
90
|
+
escaped = true;
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
if (inClass) {
|
|
94
|
+
if (ch === "]") {
|
|
95
|
+
inClass = false;
|
|
96
|
+
lastAtom = { groupRisky: false, quantified: false };
|
|
97
|
+
}
|
|
98
|
+
continue;
|
|
99
|
+
}
|
|
100
|
+
if (ch === "[") {
|
|
101
|
+
inClass = true;
|
|
102
|
+
continue;
|
|
103
|
+
}
|
|
104
|
+
if (ch === "(") {
|
|
105
|
+
groups.push({ hasQuantifier: false, hasAlternation: false });
|
|
106
|
+
lastAtom = undefined;
|
|
107
|
+
continue;
|
|
108
|
+
}
|
|
109
|
+
if (ch === ")") {
|
|
110
|
+
const group = groups.pop();
|
|
111
|
+
if (group) {
|
|
112
|
+
lastAtom = {
|
|
113
|
+
groupRisky: group.hasQuantifier || group.hasAlternation,
|
|
114
|
+
quantified: false,
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
continue;
|
|
118
|
+
}
|
|
119
|
+
if (ch === "|") {
|
|
120
|
+
const group = groups.at(-1);
|
|
121
|
+
if (group) group.hasAlternation = true;
|
|
122
|
+
lastAtom = undefined;
|
|
123
|
+
continue;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
let quantifierLength = 0;
|
|
127
|
+
if (ch === "*" || ch === "+" || ch === "?") {
|
|
128
|
+
quantifierLength = 1;
|
|
129
|
+
} else if (ch === "{") {
|
|
130
|
+
quantifierLength = /^\{\d+(?:,\d*)?\}/.exec(pattern.slice(i))?.[0].length ?? 0;
|
|
131
|
+
}
|
|
132
|
+
if (quantifierLength > 0 && lastAtom) {
|
|
133
|
+
if (ch === "?" && lastAtom.quantified) continue;
|
|
134
|
+
const variable = ch !== "{" || pattern.slice(i, i + quantifierLength).includes(",");
|
|
135
|
+
if (variable && ++variableQuantifiers > 1) unsafeRegex(pattern);
|
|
136
|
+
if (lastAtom.groupRisky) unsafeRegex(pattern);
|
|
137
|
+
const group = groups.at(-1);
|
|
138
|
+
if (group) group.hasQuantifier = true;
|
|
139
|
+
lastAtom.quantified = true;
|
|
140
|
+
i += quantifierLength - 1;
|
|
141
|
+
continue;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
lastAtom = { groupRisky: false, quantified: false };
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
55
148
|
function globToRegex(glob: string): RegExp {
|
|
149
|
+
if (glob.startsWith("/")) glob = glob.slice(1);
|
|
56
150
|
let source = "";
|
|
57
151
|
let i = 0;
|
|
58
152
|
while (i < glob.length) {
|
|
@@ -79,12 +173,6 @@ function globToRegex(glob: string): RegExp {
|
|
|
79
173
|
return new RegExp(`^${source}$`);
|
|
80
174
|
}
|
|
81
175
|
|
|
82
|
-
function isSkipableLoadError(error: unknown): boolean {
|
|
83
|
-
const code = errCode(error);
|
|
84
|
-
if (code === "EACCES" || code === "EPERM" || code === "ENOENT" || code === "ELOOP") return true;
|
|
85
|
-
return error instanceof Error && error.message.startsWith("[E_FILE_TOO_LARGE]");
|
|
86
|
-
}
|
|
87
|
-
|
|
88
176
|
interface FileHit {
|
|
89
177
|
path: string;
|
|
90
178
|
displayPath: string;
|
|
@@ -140,18 +228,24 @@ async function walkFiles(
|
|
|
140
228
|
root: string,
|
|
141
229
|
state: ScanState,
|
|
142
230
|
onFile: (absPath: string) => Promise<void>,
|
|
231
|
+
signal?: AbortSignal,
|
|
143
232
|
): Promise<void> {
|
|
144
233
|
const queue: string[] = [root];
|
|
145
|
-
|
|
146
|
-
|
|
234
|
+
let head = 0;
|
|
235
|
+
while (head < queue.length && !state.stopped) {
|
|
236
|
+
abortIf(signal);
|
|
237
|
+
const dir = queue[head++]!;
|
|
147
238
|
let entries;
|
|
148
239
|
try {
|
|
149
240
|
entries = await readdir(dir, { withFileTypes: true });
|
|
150
241
|
} catch {
|
|
151
242
|
continue;
|
|
152
243
|
}
|
|
153
|
-
|
|
244
|
+
entries.sort((a, b) => cmp(a.name, b.name));
|
|
245
|
+
for (let ei = 0; ei < entries.length; ei++) {
|
|
246
|
+
if ((ei & 127) === 0) abortIf(signal);
|
|
154
247
|
if (state.stopped) break;
|
|
248
|
+
const entry = entries[ei]!;
|
|
155
249
|
const full = join(dir, entry.name);
|
|
156
250
|
if (entry.isDirectory()) {
|
|
157
251
|
if (SKIP_DIRS.has(entry.name)) continue;
|
|
@@ -176,30 +270,20 @@ async function searchFile(
|
|
|
176
270
|
globRegex: RegExp | undefined,
|
|
177
271
|
context: number,
|
|
178
272
|
maxMatches: number,
|
|
273
|
+
signal?: AbortSignal,
|
|
179
274
|
): Promise<FileHit | undefined> {
|
|
180
275
|
const displayPath = relative(cwd, absPath).replace(/\\/g, "/");
|
|
181
276
|
if (globRegex) {
|
|
182
277
|
const globPath = relative(globRoot, absPath).replace(/\\/g, "/");
|
|
183
|
-
if (!globRegex.test(globPath)) return undefined;
|
|
184
|
-
}
|
|
185
|
-
let file;
|
|
186
|
-
try {
|
|
187
|
-
file = await loadFileKindAndText(absPath, { maxLines: MAX_HASH_LINES, displayPath });
|
|
188
|
-
} catch (error) {
|
|
189
|
-
if (isSkipableLoadError(error)) return undefined;
|
|
190
|
-
throw error;
|
|
191
|
-
}
|
|
192
|
-
if (file.kind !== "text") return undefined;
|
|
193
|
-
let norm;
|
|
194
|
-
try {
|
|
195
|
-
norm = await readNormFile(absPath, cwd, { maxLines: MAX_HASH_LINES, preloadedFile: file, noPersist: true });
|
|
196
|
-
} catch (error) {
|
|
197
|
-
if (isSkipableLoadError(error)) return undefined;
|
|
198
|
-
throw error;
|
|
278
|
+
if (!globRegex.test(globPath) && !globRegex.test(displayPath)) return undefined;
|
|
199
279
|
}
|
|
280
|
+
const norm = await tryReadNormFile(absPath, cwd, { maxLines: MAX_HASH_LINES, noPersist: true, signal });
|
|
281
|
+
if (!norm) return undefined;
|
|
200
282
|
const lines = visLines(norm.normalized);
|
|
201
283
|
const matchLines: number[] = [];
|
|
202
284
|
for (let i = 0; i < lines.length; i++) {
|
|
285
|
+
if ((i & 1023) === 0) abortIf(signal);
|
|
286
|
+
if (i !== 0 && (i & 4095) === 0) await new Promise<void>((r) => setImmediate(r));
|
|
203
287
|
if (regex.test(lines[i]!)) matchLines.push(i);
|
|
204
288
|
}
|
|
205
289
|
if (matchLines.length === 0) return undefined;
|
|
@@ -252,7 +336,7 @@ const grepToolSchema = Type.Object(
|
|
|
252
336
|
),
|
|
253
337
|
glob: Type.Optional(
|
|
254
338
|
Type.String({
|
|
255
|
-
description: "Filter files by glob pattern; * matches across directories, e.g. '*.ts' or '**/*.spec.ts'",
|
|
339
|
+
description: "Filter files by glob pattern; * matches across directories, e.g. '*.ts' or '**/*.spec.ts'. A leading / is ignored; the pattern may be relative to the search root or to the current directory.",
|
|
256
340
|
}),
|
|
257
341
|
),
|
|
258
342
|
ignoreCase: Type.Optional(
|
|
@@ -290,6 +374,7 @@ export function regGrep(pi: ExtensionAPI): void {
|
|
|
290
374
|
promptGuidelines: loadGuide("../prompts/grep-guidelines.md"),
|
|
291
375
|
prepareArguments: makePrepareArguments(),
|
|
292
376
|
parameters: grepToolSchema,
|
|
377
|
+
executionMode: "sequential",
|
|
293
378
|
|
|
294
379
|
async execute(_toolCallId, params, signal, _onUpdate, ctx) {
|
|
295
380
|
const canonical = normReq(params);
|
|
@@ -318,7 +403,8 @@ export function regGrep(pi: ExtensionAPI): void {
|
|
|
318
403
|
} else {
|
|
319
404
|
await walkFiles(base, state, async (absPath) => {
|
|
320
405
|
files.push(absPath);
|
|
321
|
-
});
|
|
406
|
+
}, signal);
|
|
407
|
+
files.sort(cmp);
|
|
322
408
|
}
|
|
323
409
|
const hits: FileHit[] = [];
|
|
324
410
|
let matches = 0;
|
|
@@ -335,7 +421,7 @@ export function regGrep(pi: ExtensionAPI): void {
|
|
|
335
421
|
abortIf(signal);
|
|
336
422
|
const absPath = files[f]!;
|
|
337
423
|
if (countOnly) {
|
|
338
|
-
const hit = await searchFile(absPath, globRoot, ctx.cwd, regex, globRegex, context, Number.MAX_SAFE_INTEGER);
|
|
424
|
+
const hit = await searchFile(absPath, globRoot, ctx.cwd, regex, globRegex, context, Number.MAX_SAFE_INTEGER, signal);
|
|
339
425
|
if (!hit) continue;
|
|
340
426
|
totalRows += hit.rows.length;
|
|
341
427
|
for (const row of hit.rows) totalBytes += Buffer.byteLength(row, "utf-8") + 1;
|
|
@@ -353,7 +439,7 @@ export function regGrep(pi: ExtensionAPI): void {
|
|
|
353
439
|
limitTruncated = true;
|
|
354
440
|
break;
|
|
355
441
|
}
|
|
356
|
-
const hit = await searchFile(absPath, globRoot, ctx.cwd, regex, globRegex, context, remaining);
|
|
442
|
+
const hit = await searchFile(absPath, globRoot, ctx.cwd, regex, globRegex, context, remaining, signal);
|
|
357
443
|
if (!hit) continue;
|
|
358
444
|
const keptRows: string[] = [];
|
|
359
445
|
const keptHashes: string[] = [];
|
|
@@ -382,6 +468,7 @@ export function regGrep(pi: ExtensionAPI): void {
|
|
|
382
468
|
hits.push({ ...hit, rows: keptRows, hashes: keptHashes });
|
|
383
469
|
if (rowTruncated) countOnly = true;
|
|
384
470
|
}
|
|
471
|
+
hits.sort((a, b) => cmp(a.displayPath, b.displayPath));
|
|
385
472
|
for (const hit of hits) {
|
|
386
473
|
await recordServedSafe(hit.path, hit.hashes, "grep", new Set(hit.fileHashes));
|
|
387
474
|
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
export interface SnapshotCacheEntry {
|
|
2
|
+
checksum: string;
|
|
3
|
+
lineCount: number;
|
|
4
|
+
hashes: string[];
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
export const SNAPSHOT_CACHE_LIMIT = 256;
|
|
8
|
+
|
|
9
|
+
export const snapshotCache = new Map<string, SnapshotCacheEntry>();
|
|
10
|
+
|
|
11
|
+
export function cacheSnapshot(path: string, checksum: string, lineCount: number, hashes: string[]): void {
|
|
12
|
+
snapshotCache.delete(path);
|
|
13
|
+
snapshotCache.set(path, { checksum, lineCount, hashes: hashes.slice() });
|
|
14
|
+
if (snapshotCache.size > SNAPSHOT_CACHE_LIMIT) {
|
|
15
|
+
const oldest = snapshotCache.keys().next().value;
|
|
16
|
+
if (oldest !== undefined) snapshotCache.delete(oldest);
|
|
17
|
+
}
|
|
18
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { isBusyError } from "./validation";
|
|
2
|
+
|
|
3
|
+
const sleepSab = new Int32Array(new SharedArrayBuffer(4));
|
|
4
|
+
|
|
5
|
+
function sleepSync(ms: number): void {
|
|
6
|
+
Atomics.wait(sleepSab, 0, 0, ms);
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
const BUSY_RETRIES = 3;
|
|
10
|
+
const BUSY_RETRY_DELAY_MS = 50;
|
|
11
|
+
|
|
12
|
+
export function withBusyRetry<T>(fn: () => T): T {
|
|
13
|
+
let lastError: unknown;
|
|
14
|
+
for (let attempt = 0; attempt <= BUSY_RETRIES; attempt++) {
|
|
15
|
+
try {
|
|
16
|
+
return fn();
|
|
17
|
+
} catch (error) {
|
|
18
|
+
lastError = error;
|
|
19
|
+
if (!isBusyError(error) || attempt === BUSY_RETRIES) throw error;
|
|
20
|
+
sleepSync(BUSY_RETRY_DELAY_MS * (1 << attempt));
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
throw lastError;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export async function withBusyRetryAsync<T>(fn: () => T): Promise<T> {
|
|
27
|
+
let lastError: unknown;
|
|
28
|
+
for (let attempt = 0; attempt <= BUSY_RETRIES; attempt++) {
|
|
29
|
+
try {
|
|
30
|
+
return fn();
|
|
31
|
+
} catch (error) {
|
|
32
|
+
lastError = error;
|
|
33
|
+
if (!isBusyError(error) || attempt === BUSY_RETRIES) throw error;
|
|
34
|
+
await new Promise<void>((r) => setTimeout(r, BUSY_RETRY_DELAY_MS * (1 << attempt)));
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
throw lastError;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export function retriedWrite(stmt: { run(...params: (string | number)[]): unknown }): (...params: (string | number)[]) => void {
|
|
41
|
+
return (...params) => {
|
|
42
|
+
withBusyRetry(() => { stmt.run(...params); });
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export async function openDbWithBusyRetryAsync<T>(fn: () => T): Promise<T> {
|
|
47
|
+
return withBusyRetryAsync(fn);
|
|
48
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { HASH_RE } from "../hashline/alphabet";
|
|
2
|
+
|
|
3
|
+
export function isValidHashList(value: unknown): value is string[] {
|
|
4
|
+
if (!Array.isArray(value)) return false;
|
|
5
|
+
for (const hash of value) {
|
|
6
|
+
if (typeof hash !== "string" || !HASH_RE.test(hash)) return false;
|
|
7
|
+
}
|
|
8
|
+
if (new Set(value).size !== value.length) return false;
|
|
9
|
+
return true;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export function parseHashList(raw: string, onInvalid: () => void, context?: string): string[] | undefined {
|
|
13
|
+
let parsed: unknown;
|
|
14
|
+
try {
|
|
15
|
+
parsed = JSON.parse(raw);
|
|
16
|
+
} catch (error) {
|
|
17
|
+
console.error(`[parseHashList]${context ? ` ${context}:` : ""} failed to parse stored hashes JSON:`, error);
|
|
18
|
+
onInvalid();
|
|
19
|
+
return undefined;
|
|
20
|
+
}
|
|
21
|
+
if (!isValidHashList(parsed)) {
|
|
22
|
+
console.error(`[parseHashList]${context ? ` ${context}:` : ""} stored hashes did not pass validation:`, Array.isArray(parsed) ? `length=${parsed.length} sample=${JSON.stringify(parsed.slice(0, 3))}` : (() => { try { return JSON.stringify(parsed)?.slice(0, 500) ?? String(parsed).slice(0, 500); } catch { return String(parsed).slice(0, 500); } })());
|
|
23
|
+
onInvalid();
|
|
24
|
+
return undefined;
|
|
25
|
+
}
|
|
26
|
+
return parsed;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export function parseStoredHashes(row: Record<string, unknown> | undefined, onInvalid: () => void): string[] | undefined {
|
|
30
|
+
if (!row) return undefined;
|
|
31
|
+
return parseHashList(row.hashes as string, onInvalid);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function isValidSnapshot(value: unknown): value is { content: string; hashes: string[] } {
|
|
35
|
+
if (typeof value !== "object" || value === null) return false;
|
|
36
|
+
const v = value as Record<string, unknown>;
|
|
37
|
+
if (typeof v.content !== "string") return false;
|
|
38
|
+
return isValidHashList(v.hashes);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export function isCorruptionError(error: unknown): boolean {
|
|
42
|
+
if (error && typeof error === "object") {
|
|
43
|
+
const errcode = (error as { errcode?: unknown }).errcode;
|
|
44
|
+
if (typeof errcode === "number") {
|
|
45
|
+
return errcode === 11 || errcode === 24 || errcode === 26;
|
|
46
|
+
}
|
|
47
|
+
const code = (error as { code?: unknown }).code;
|
|
48
|
+
if (typeof code === "string" && /NOTADB|CORRUPT/.test(code)) return true;
|
|
49
|
+
}
|
|
50
|
+
return (
|
|
51
|
+
error instanceof Error &&
|
|
52
|
+
/corrupt|not a database|malformed|database disk image/i.test(error.message)
|
|
53
|
+
);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export function isBusyError(error: unknown): boolean {
|
|
57
|
+
if (error && typeof error === "object") {
|
|
58
|
+
const errcode = (error as { errcode?: unknown }).errcode;
|
|
59
|
+
if (typeof errcode === "number") return errcode === 5 || errcode === 6;
|
|
60
|
+
}
|
|
61
|
+
return error instanceof Error && /busy|locked/i.test(error.message);
|
|
62
|
+
}
|
package/src/hash-store.ts
CHANGED
|
@@ -1,10 +1,30 @@
|
|
|
1
1
|
import { existsSync } from "fs";
|
|
2
|
-
import { readFile, rename, mkdir, stat } from "fs/promises";
|
|
2
|
+
import { chmod, readFile, rename, mkdir, stat } from "fs/promises";
|
|
3
3
|
import { hashStorePath, hashStoreDir, legacyHashStorePath } from "./paths";
|
|
4
4
|
import { errCode, isRec, splitLines } from "./utils";
|
|
5
5
|
import { initHasher, contentChecksum } from "./hashline/hasher";
|
|
6
|
-
import { HASH_RE } from "./hashline/alphabet";
|
|
7
6
|
import { HASH_STORE_VERSION, HASH_STORE_BUSY_TIMEOUT } from "./constants";
|
|
7
|
+
import {
|
|
8
|
+
isValidHashList,
|
|
9
|
+
parseStoredHashes,
|
|
10
|
+
isValidSnapshot,
|
|
11
|
+
isCorruptionError,
|
|
12
|
+
parseHashList,
|
|
13
|
+
} from "./hash-store/validation";
|
|
14
|
+
import {
|
|
15
|
+
withBusyRetry,
|
|
16
|
+
retriedWrite,
|
|
17
|
+
openDbWithBusyRetryAsync,
|
|
18
|
+
} from "./hash-store/retry";
|
|
19
|
+
import {
|
|
20
|
+
snapshotCache,
|
|
21
|
+
cacheSnapshot,
|
|
22
|
+
SNAPSHOT_CACHE_LIMIT,
|
|
23
|
+
} from "./hash-store/cache";
|
|
24
|
+
|
|
25
|
+
export { isValidHashList, parseHashList, parseStoredHashes, isCorruptionError };
|
|
26
|
+
export { SNAPSHOT_CACHE_LIMIT };
|
|
27
|
+
export const STORE_NOT_OPEN_MESSAGE = "Hash store is not open; transactional update aborted";
|
|
8
28
|
|
|
9
29
|
type SqlParams = (string | number)[];
|
|
10
30
|
|
|
@@ -90,117 +110,10 @@ export interface UndoRecord {
|
|
|
90
110
|
resultContent: string;
|
|
91
111
|
}
|
|
92
112
|
|
|
93
|
-
interface LegacySnapshot {
|
|
94
|
-
content: string;
|
|
95
|
-
hashes: string[];
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
export function isValidHashList(value: unknown): value is string[] {
|
|
99
|
-
if (!Array.isArray(value)) return false;
|
|
100
|
-
for (const hash of value) {
|
|
101
|
-
if (typeof hash !== "string" || !HASH_RE.test(hash)) return false;
|
|
102
|
-
}
|
|
103
|
-
if (new Set(value).size !== value.length) return false;
|
|
104
|
-
return true;
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
export function parseHashList(raw: string, onInvalid: () => void): string[] | undefined {
|
|
108
|
-
let parsed: unknown;
|
|
109
|
-
try {
|
|
110
|
-
parsed = JSON.parse(raw);
|
|
111
|
-
} catch {
|
|
112
|
-
onInvalid();
|
|
113
|
-
return undefined;
|
|
114
|
-
}
|
|
115
|
-
if (!isValidHashList(parsed)) {
|
|
116
|
-
onInvalid();
|
|
117
|
-
return undefined;
|
|
118
|
-
}
|
|
119
|
-
return parsed;
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
export function parseStoredHashes(
|
|
123
|
-
row: Record<string, unknown> | undefined,
|
|
124
|
-
onInvalid: () => void,
|
|
125
|
-
): string[] | undefined {
|
|
126
|
-
if (!row) return undefined;
|
|
127
|
-
return parseHashList(row.hashes as string, onInvalid);
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
function isValidSnapshot(value: unknown): value is LegacySnapshot {
|
|
131
|
-
if (typeof value !== "object" || value === null) return false;
|
|
132
|
-
const v = value as Record<string, unknown>;
|
|
133
|
-
if (typeof v.content !== "string") return false;
|
|
134
|
-
return isValidHashList(v.hashes);
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
export function isCorruptionError(error: unknown): boolean {
|
|
138
|
-
if (error && typeof error === "object") {
|
|
139
|
-
const errcode = (error as { errcode?: unknown }).errcode;
|
|
140
|
-
if (typeof errcode === "number") {
|
|
141
|
-
return errcode === 11 || errcode === 24 || errcode === 26;
|
|
142
|
-
}
|
|
143
|
-
const code = (error as { code?: unknown }).code;
|
|
144
|
-
if (typeof code === "string" && /NOTADB|CORRUPT/.test(code)) return true;
|
|
145
|
-
}
|
|
146
|
-
return (
|
|
147
|
-
error instanceof Error &&
|
|
148
|
-
/corrupt|not a database|malformed|database disk image/i.test(error.message)
|
|
149
|
-
);
|
|
150
|
-
}
|
|
151
|
-
|
|
152
|
-
function isBusyError(error: unknown): boolean {
|
|
153
|
-
if (error && typeof error === "object") {
|
|
154
|
-
const errcode = (error as { errcode?: unknown }).errcode;
|
|
155
|
-
if (typeof errcode === "number") return errcode === 5 || errcode === 6;
|
|
156
|
-
}
|
|
157
|
-
return error instanceof Error && /busy|locked/i.test(error.message);
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
function sleepSync(ms: number): void {
|
|
161
|
-
const sab = new Int32Array(new SharedArrayBuffer(4));
|
|
162
|
-
Atomics.wait(sab, 0, 0, ms);
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
const BUSY_RETRIES = 3;
|
|
166
|
-
const BUSY_RETRY_DELAY_MS = 100;
|
|
167
|
-
|
|
168
|
-
function withBusyRetry<T>(fn: () => T): T {
|
|
169
|
-
let lastError: unknown;
|
|
170
|
-
for (let attempt = 0; attempt <= BUSY_RETRIES; attempt++) {
|
|
171
|
-
try {
|
|
172
|
-
return fn();
|
|
173
|
-
} catch (error) {
|
|
174
|
-
lastError = error;
|
|
175
|
-
if (!isBusyError(error) || attempt === BUSY_RETRIES) throw error;
|
|
176
|
-
sleepSync(BUSY_RETRY_DELAY_MS);
|
|
177
|
-
}
|
|
178
|
-
}
|
|
179
|
-
throw lastError;
|
|
180
|
-
}
|
|
181
|
-
|
|
182
|
-
function openDbWithBusyRetry(storePath: string): { db: RawDb; stmts: Prepared } {
|
|
183
|
-
return withBusyRetry(() => openDb(storePath));
|
|
184
|
-
}
|
|
185
|
-
|
|
186
|
-
function retriedWrite(
|
|
187
|
-
stmt: { run(...params: SqlParams): unknown },
|
|
188
|
-
): (...params: SqlParams) => void {
|
|
189
|
-
return (...params) => {
|
|
190
|
-
withBusyRetry(() => { stmt.run(...params); });
|
|
191
|
-
};
|
|
192
|
-
}
|
|
193
|
-
|
|
194
113
|
let cachedDb: { path: string; db: RawDb; stmts: Prepared } | null = null;
|
|
195
114
|
let opening: { path: string; promise: Promise<HashStore> } | null = null;
|
|
196
115
|
let exitHandlerRegistered = false;
|
|
197
|
-
|
|
198
|
-
checksum: string;
|
|
199
|
-
lineCount: number;
|
|
200
|
-
hashes: string[];
|
|
201
|
-
}
|
|
202
|
-
const snapshotCache = new Map<string, SnapshotCacheEntry>();
|
|
203
|
-
export const SNAPSHOT_CACHE_LIMIT = 256;
|
|
116
|
+
|
|
204
117
|
function openDb(storePath: string): { db: RawDb; stmts: Prepared } {
|
|
205
118
|
const db = openDbFn(storePath);
|
|
206
119
|
try {
|
|
@@ -213,9 +126,7 @@ function openDb(storePath: string): { db: RawDb; stmts: Prepared } {
|
|
|
213
126
|
}
|
|
214
127
|
}
|
|
215
128
|
|
|
216
|
-
function buildStore(
|
|
217
|
-
db: RawDb,
|
|
218
|
-
): { db: RawDb; stmts: Prepared } {
|
|
129
|
+
function buildStore(db: RawDb): { db: RawDb; stmts: Prepared } {
|
|
219
130
|
db.exec("PRAGMA journal_mode = WAL");
|
|
220
131
|
db.exec("PRAGMA synchronous = NORMAL");
|
|
221
132
|
db.exec(
|
|
@@ -333,30 +244,45 @@ function shutdownDb(db: RawDb): void {
|
|
|
333
244
|
}
|
|
334
245
|
|
|
335
246
|
async function openStore(storePath: string): Promise<HashStore> {
|
|
336
|
-
|
|
337
|
-
|
|
247
|
+
if (cachedDb && cachedDb.path === storePath && cachedDb.db.isOpen) {
|
|
248
|
+
return { stmts: cachedDb.stmts, engine: sqliteEngine };
|
|
249
|
+
}
|
|
250
|
+
if (cachedDb) shutdownHashStore();
|
|
338
251
|
await initHasher();
|
|
339
|
-
await mkdir(hashStoreDir(), { recursive: true });
|
|
252
|
+
await mkdir(hashStoreDir(), { recursive: true, mode: 0o700 });
|
|
253
|
+
if (process.platform !== "win32") {
|
|
254
|
+
await chmod(hashStoreDir(), 0o700);
|
|
255
|
+
}
|
|
340
256
|
|
|
341
257
|
let existed = existsSync(storePath);
|
|
342
258
|
let opened: { db: RawDb; stmts: Prepared };
|
|
343
259
|
try {
|
|
344
|
-
opened =
|
|
260
|
+
opened = await openDbWithBusyRetryAsync(() => openDb(storePath));
|
|
345
261
|
} catch (error) {
|
|
346
262
|
if (!isCorruptionError(error)) throw error;
|
|
347
263
|
console.error("Hash store failed to open, rebuilding:", error);
|
|
348
264
|
await quarantineStore(storePath);
|
|
349
265
|
existed = false;
|
|
350
|
-
opened =
|
|
266
|
+
opened = await openDbWithBusyRetryAsync(() => openDb(storePath));
|
|
351
267
|
}
|
|
352
268
|
if (!isHealthy(opened.db)) {
|
|
353
269
|
shutdownDb(opened.db);
|
|
354
270
|
await quarantineStore(storePath);
|
|
355
271
|
existed = false;
|
|
356
|
-
opened =
|
|
272
|
+
opened = await openDbWithBusyRetryAsync(() => openDb(storePath));
|
|
357
273
|
}
|
|
358
274
|
const { db, stmts } = opened;
|
|
359
275
|
|
|
276
|
+
if (process.platform !== "win32") {
|
|
277
|
+
for (const candidate of [storePath, `${storePath}-wal`, `${storePath}-shm`]) {
|
|
278
|
+
try {
|
|
279
|
+
await chmod(candidate, 0o600);
|
|
280
|
+
} catch (error) {
|
|
281
|
+
if (errCode(error) !== "ENOENT") throw error;
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
|
|
360
286
|
if (!existed) {
|
|
361
287
|
try {
|
|
362
288
|
await migrateLegacy(db);
|
|
@@ -403,9 +329,9 @@ export function shutdownHashStore(): void {
|
|
|
403
329
|
snapshotCache.clear();
|
|
404
330
|
}
|
|
405
331
|
|
|
406
|
-
function withStore(fn: () => void): void {
|
|
407
|
-
if (!cachedDb) {
|
|
408
|
-
throw new Error(
|
|
332
|
+
export function withStore(fn: () => void): void {
|
|
333
|
+
if (!cachedDb || !cachedDb.db.isOpen) {
|
|
334
|
+
throw new Error(STORE_NOT_OPEN_MESSAGE);
|
|
409
335
|
}
|
|
410
336
|
withBusyRetry(() => {
|
|
411
337
|
cachedDb!.db.exec("BEGIN IMMEDIATE");
|
|
@@ -485,15 +411,6 @@ async function migrateLegacy(db: RawDb): Promise<void> {
|
|
|
485
411
|
}
|
|
486
412
|
}
|
|
487
413
|
|
|
488
|
-
function cacheSnapshot(path: string, checksum: string, lineCount: number, hashes: string[]): void {
|
|
489
|
-
snapshotCache.delete(path);
|
|
490
|
-
snapshotCache.set(path, { checksum, lineCount, hashes: hashes.slice() });
|
|
491
|
-
if (snapshotCache.size > SNAPSHOT_CACHE_LIMIT) {
|
|
492
|
-
const oldest = snapshotCache.keys().next().value;
|
|
493
|
-
if (oldest !== undefined) snapshotCache.delete(oldest);
|
|
494
|
-
}
|
|
495
|
-
}
|
|
496
|
-
|
|
497
414
|
export function getSnapshot(
|
|
498
415
|
store: HashStore,
|
|
499
416
|
path: string,
|
|
@@ -528,6 +445,14 @@ export function upsertSnapshot(
|
|
|
528
445
|
store.stmts.upsert(path, checksum, lineCount, JSON.stringify(hashes), Date.now());
|
|
529
446
|
cacheSnapshot(path, checksum, lineCount, hashes);
|
|
530
447
|
}
|
|
448
|
+
export function persistSnapshot(
|
|
449
|
+
store: HashStore,
|
|
450
|
+
path: string,
|
|
451
|
+
content: string,
|
|
452
|
+
hashes: string[],
|
|
453
|
+
): void {
|
|
454
|
+
upsertSnapshot(store, path, contentChecksum(content), splitLines(content).length, hashes);
|
|
455
|
+
}
|
|
531
456
|
|
|
532
457
|
export function upsertUndo(store: HashStore, path: string, entry: UndoRecord): void {
|
|
533
458
|
store.stmts.undoUpsert(
|
|
@@ -593,22 +518,32 @@ export async function pruneMissing(store: HashStore): Promise<void> {
|
|
|
593
518
|
withStore(() => {
|
|
594
519
|
for (const path of missing) {
|
|
595
520
|
store.stmts.deleteOne(path);
|
|
596
|
-
snapshotCache.delete(path);
|
|
597
521
|
store.stmts.servedDelete(path);
|
|
598
522
|
}
|
|
599
523
|
});
|
|
524
|
+
for (const path of missing) snapshotCache.delete(path);
|
|
600
525
|
}
|
|
601
526
|
|
|
602
527
|
function matchPathsByHashes(
|
|
603
528
|
rows: { path: string; hashes: string }[],
|
|
604
529
|
hashes: string[],
|
|
605
530
|
): string[] {
|
|
531
|
+
const needed = new Set(hashes);
|
|
532
|
+
if (needed.size === 0) return [];
|
|
606
533
|
const matches: string[] = [];
|
|
607
534
|
for (const row of rows) {
|
|
608
535
|
try {
|
|
609
536
|
const parsed = JSON.parse(row.hashes) as unknown;
|
|
610
537
|
if (!isValidHashList(parsed)) continue;
|
|
611
|
-
|
|
538
|
+
const parsedSet = new Set(parsed);
|
|
539
|
+
let ok = true;
|
|
540
|
+
for (const h of needed) {
|
|
541
|
+
if (!parsedSet.has(h)) {
|
|
542
|
+
ok = false;
|
|
543
|
+
break;
|
|
544
|
+
}
|
|
545
|
+
}
|
|
546
|
+
if (ok) matches.push(row.path);
|
|
612
547
|
} catch {
|
|
613
548
|
continue;
|
|
614
549
|
}
|