sliftutils 1.7.32 → 1.7.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/builders/extensionBuild.ts +1 -1
- package/bundler/bundleEntry.ts +1 -3
- package/bundler/bundler.ts +2 -4
- package/bundler/sourceMaps.ts +1 -2
- package/examplestorage/browser.tsx +1 -2
- package/examplestorage/exampleserver.ts +1 -2
- package/index.d.ts +134 -102
- package/misc/dist/environment.ts.cache +2 -2
- package/misc/dist/getSecret.ts.cache +2 -2
- package/misc/dist/strings.ts.cache +2 -2
- package/misc/dist/zip.ts.cache +2 -2
- package/misc/https/dist/certs.ts.cache +2 -2
- package/misc/https/dist/dns.ts.cache +7 -16
- package/misc/https/dist/persistentLocalStorage.ts.cache +2 -2
- package/misc/https/dns.d.ts +1 -4
- package/misc/https/dns.ts +5 -14
- package/misc/https/hostServer.d.ts +2 -0
- package/misc/https/hostServer.ts +6 -7
- package/misc/https/httpsCerts.ts +4 -10
- package/misc/openrouter.ts +1 -2
- package/package.json +1 -1
- package/render-utils/FullscreenModal.tsx +1 -4
- package/render-utils/GenericFormat.tsx +2 -8
- package/render-utils/Input.tsx +5 -13
- package/render-utils/InputLabel.tsx +4 -6
- package/render-utils/InputPicker.tsx +1 -2
- package/render-utils/URLParam.ts +2 -3
- package/render-utils/dist/observer.tsx.cache +2 -2
- package/render-utils/mobxTyped.ts +2 -3
- package/render-utils/niceStringify.ts +2 -4
- package/storage/ArchivesDisk.ts +4 -14
- package/storage/BulkDatabase2/BulkDatabase2.d.ts +1 -2
- package/storage/BulkDatabase2/BulkDatabase2.ts +4 -11
- package/storage/BulkDatabase2/BulkDatabaseBase.ts +42 -114
- package/storage/BulkDatabase2/BulkDatabaseFormat.ts +31 -91
- package/storage/BulkDatabase2/BulkDatabaseMerge.ts +19 -57
- package/storage/BulkDatabase2/BulkDatabaseReader.ts +4 -12
- package/storage/BulkDatabase2/LoadedIndex.ts +6 -18
- package/storage/BulkDatabase2/WriteOverlay.ts +3 -8
- package/storage/BulkDatabase2/blockCache.ts +12 -28
- package/storage/BulkDatabase2/dist/BulkDatabaseBase.ts.cache +40 -103
- package/storage/BulkDatabase2/dist/BulkDatabaseFormat.ts.cache +19 -50
- package/storage/BulkDatabase2/dist/BulkDatabaseMerge.ts.cache +18 -50
- package/storage/BulkDatabase2/dist/BulkDatabaseReader.ts.cache +7 -15
- package/storage/BulkDatabase2/dist/LoadedIndex.ts.cache +9 -21
- package/storage/BulkDatabase2/dist/WriteOverlay.ts.cache +6 -11
- package/storage/BulkDatabase2/dist/blockCache.ts.cache +15 -31
- package/storage/BulkDatabase2/dist/mergeLock.ts.cache +10 -29
- package/storage/BulkDatabase2/dist/mergeMarkers.ts.cache +10 -18
- package/storage/BulkDatabase2/dist/streamLog.ts.cache +17 -43
- package/storage/BulkDatabase2/dist/syncClient.ts.cache +11 -27
- package/storage/BulkDatabase2/mergeLock.ts +7 -26
- package/storage/BulkDatabase2/mergeMarkers.ts +7 -15
- package/storage/BulkDatabase2/streamLog.ts +14 -40
- package/storage/BulkDatabase2/syncClient.ts +8 -24
- package/storage/FileFolderAPI.tsx +28 -75
- package/storage/IArchives.d.ts +3 -3
- package/storage/IArchives.ts +21 -50
- package/storage/IStorage.ts +2 -4
- package/storage/PendingManager.tsx +2 -3
- package/storage/TransactionStorage.ts +11 -24
- package/storage/backblaze.ts +12 -43
- package/storage/dist/ArchivesDisk.ts.cache +7 -17
- package/storage/dist/FileFolderAPI.tsx.cache +25 -63
- package/storage/dist/IArchives.ts.cache +10 -20
- package/storage/dist/IndexedDBFileFolderAPI.ts.cache +2 -2
- package/storage/dist/JSONStorage.ts.cache +2 -2
- package/storage/dist/PendingManager.tsx.cache +5 -6
- package/storage/dist/TransactionStorage.ts.cache +14 -27
- package/storage/dist/backblaze.ts.cache +15 -46
- package/storage/dist/embeddingFormats.ts.cache +11 -24
- package/storage/dist/fileSystemPointer.ts.cache +4 -7
- package/storage/dist/remoteFileStorage.ts.cache +17 -36
- package/storage/embeddingFormats.ts +11 -27
- package/storage/fileSystemPointer.ts +2 -7
- package/storage/proxydatabase/dist/ivfEmbeddingDatabase.ts.cache +8 -18
- package/storage/proxydatabase/dist/transactionSet.ts.cache +4 -7
- package/storage/proxydatabase/inMemoryDatabase.ts +1 -3
- package/storage/proxydatabase/ivfEmbeddingDatabase.ts +8 -23
- package/storage/proxydatabase/transactionSet.ts +2 -5
- package/storage/remoteFileServer.ts +20 -48
- package/storage/remoteFileStorage.ts +14 -33
- package/storage/remoteStorage/ArchivesRemote.d.ts +1 -4
- package/storage/remoteStorage/ArchivesRemote.ts +11 -26
- package/storage/remoteStorage/ArchivesUrl.ts +2 -4
- package/storage/remoteStorage/accessPage.tsx +1 -5
- package/storage/remoteStorage/blobStore.d.ts +5 -13
- package/storage/remoteStorage/blobStore.ts +52 -130
- package/storage/remoteStorage/createArchives.d.ts +22 -14
- package/storage/remoteStorage/createArchives.ts +54 -188
- package/storage/remoteStorage/deployTakeover.d.ts +18 -32
- package/storage/remoteStorage/deployTakeover.ts +70 -384
- package/storage/remoteStorage/dist/ArchivesRemote.ts.cache +12 -24
- package/storage/remoteStorage/dist/ArchivesUrl.ts.cache +5 -7
- package/storage/remoteStorage/dist/blobStore.ts.cache +47 -117
- package/storage/remoteStorage/dist/cliArgs.ts.cache +2 -2
- package/storage/remoteStorage/dist/createArchives.ts.cache +58 -189
- package/storage/remoteStorage/dist/deployTakeover.ts.cache +81 -387
- package/storage/remoteStorage/dist/intermediateSources.ts.cache +119 -0
- package/storage/remoteStorage/dist/remoteConfig.ts.cache +36 -24
- package/storage/remoteStorage/dist/sourceWrapper.ts.cache +20 -40
- package/storage/remoteStorage/dist/storageClientController.ts.cache +4 -6
- package/storage/remoteStorage/dist/storageController.ts.cache +37 -74
- package/storage/remoteStorage/dist/storageServerState.ts.cache +345 -188
- package/storage/remoteStorage/grantAccessCli.ts +1 -3
- package/storage/remoteStorage/intermediateSources.d.ts +21 -0
- package/storage/remoteStorage/intermediateSources.ts +117 -0
- package/storage/remoteStorage/remoteConfig.ts +33 -20
- package/storage/remoteStorage/sourceWrapper.d.ts +8 -19
- package/storage/remoteStorage/sourceWrapper.ts +19 -41
- package/storage/remoteStorage/storageClientController.ts +1 -3
- package/storage/remoteStorage/storageController.d.ts +7 -6
- package/storage/remoteStorage/storageController.ts +36 -82
- package/storage/remoteStorage/storageServer.ts +20 -34
- package/storage/remoteStorage/storageServerState.d.ts +41 -5
- package/storage/remoteStorage/storageServerState.ts +355 -216
|
@@ -19,13 +19,9 @@ process.on("unhandledRejection", reason => {
|
|
|
19
19
|
console.error(`[writeServer] unhandledRejection (continuing):`, (reason as Error)?.stack ?? reason);
|
|
20
20
|
});
|
|
21
21
|
|
|
22
|
-
// Remote file server for sliftutils getRemoteFileStorage / BulkDatabase2. Serves one folder on disk over
|
|
23
|
-
//
|
|
24
|
-
//
|
|
25
|
-
//
|
|
26
|
-
// SECURITY: the cert is self-signed, so a browser must accept it once (open the printed https URL and
|
|
27
|
-
// click through), and the Node client connects with cert verification disabled. The password is what
|
|
28
|
-
// authorizes access; keep it secret.
|
|
22
|
+
// Remote file server for sliftutils getRemoteFileStorage / BulkDatabase2. Serves one folder on disk over self-signed HTTPS, authenticated with an auto-generated 6-word password (sent as `Authorization: Bearer <password>`). Run it with `yarn filehoster <folder>` (or the `filehoster` bin).
|
|
23
|
+
//
|
|
24
|
+
// SECURITY: the cert is self-signed, so a browser must accept it once (open the printed https URL and click through), and the Node client connects with cert verification disabled. The password is what authorizes access; keep it secret.
|
|
29
25
|
|
|
30
26
|
// Common, memorable words (frequency-ordered subset) for passphrase generation.
|
|
31
27
|
const WORDS: string[] = [
|
|
@@ -207,16 +203,12 @@ export function generatePassword(wordCount: number): string {
|
|
|
207
203
|
return words.join(" ");
|
|
208
204
|
}
|
|
209
205
|
|
|
210
|
-
// The password is always a list of words, so we compare case-insensitively and ignore any non-letter
|
|
211
|
-
// characters. That makes it forgiving of voice dictation and mobile autocorrect (spacing, capitals,
|
|
212
|
-
// stray punctuation) without meaningfully weakening a 6-word passphrase.
|
|
206
|
+
// The password is always a list of words, so we compare case-insensitively and ignore any non-letter characters. That makes it forgiving of voice dictation and mobile autocorrect (spacing, capitals, stray punctuation) without meaningfully weakening a 6-word passphrase.
|
|
213
207
|
function normalizePassword(p: string): string {
|
|
214
208
|
return String(p || "").toLowerCase().replace(/[^a-z]/g, "");
|
|
215
209
|
}
|
|
216
210
|
|
|
217
|
-
// The password is generated once and saved (next to the cert, outside the served folder), then reused
|
|
218
|
-
// across restarts — so connected clients keep working after the server bounces, instead of getting a
|
|
219
|
-
// new password every time. Override with the PASSWORD env var.
|
|
211
|
+
// The password is generated once and saved (next to the cert, outside the served folder), then reused across restarts — so connected clients keep working after the server bounces, instead of getting a new password every time. Override with the PASSWORD env var.
|
|
220
212
|
function getOrCreatePassword(): string {
|
|
221
213
|
const dir = path.join(os.homedir(), ".sliftutils-remote");
|
|
222
214
|
fs.mkdirSync(dir, { recursive: true });
|
|
@@ -230,8 +222,7 @@ function getOrCreatePassword(): string {
|
|
|
230
222
|
return pw;
|
|
231
223
|
}
|
|
232
224
|
|
|
233
|
-
// Generate (once) and cache a self-signed key + cert via openssl, outside the served folder so its
|
|
234
|
-
// private key is never reachable through the API.
|
|
225
|
+
// Generate (once) and cache a self-signed key + cert via openssl, outside the served folder so its private key is never reachable through the API.
|
|
235
226
|
function getCert(): { key: Buffer; cert: Buffer } {
|
|
236
227
|
const dir = path.join(os.homedir(), ".sliftutils-remote");
|
|
237
228
|
fs.mkdirSync(dir, { recursive: true });
|
|
@@ -284,9 +275,7 @@ function formatBytes(n: number): string {
|
|
|
284
275
|
return (n / 1024 / 1024 / 1024).toFixed(2) + "GB";
|
|
285
276
|
}
|
|
286
277
|
|
|
287
|
-
// Parse a single-range HTTP Range header ("bytes=start-end", "bytes=start-", "bytes=-suffix") into an
|
|
288
|
-
// INCLUSIVE [start, end]. Returns "unsatisfiable" for a range past the file, or undefined for no/garbled
|
|
289
|
-
// range (caller then serves the whole file). Only single ranges are supported (what media players use).
|
|
278
|
+
// Parse a single-range HTTP Range header ("bytes=start-end", "bytes=start-", "bytes=-suffix") into an INCLUSIVE [start, end]. Returns "unsatisfiable" for a range past the file, or undefined for no/garbled range (caller then serves the whole file). Only single ranges are supported (what media players use).
|
|
290
279
|
function parseRange(header: string, size: number): { start: number; end: number } | "unsatisfiable" | undefined {
|
|
291
280
|
const m = /^bytes=(\d*)-(\d*)$/.exec(header.trim());
|
|
292
281
|
if (!m) return undefined;
|
|
@@ -318,9 +307,7 @@ function mediaContentType(filePath: string): string {
|
|
|
318
307
|
return MEDIA_TYPES[ext] || "application/octet-stream";
|
|
319
308
|
}
|
|
320
309
|
|
|
321
|
-
// Public (no-auth) landing page. The browser can't trust a self-signed cert from a background fetch, so
|
|
322
|
-
// the user opens this URL once, accepts the browser's security warning, and the cert becomes trusted for
|
|
323
|
-
// the origin — then the app's fetches work. This page is what they see after accepting.
|
|
310
|
+
// Public (no-auth) landing page. The browser can't trust a self-signed cert from a background fetch, so the user opens this URL once, accepts the browser's security warning, and the cert becomes trusted for the origin — then the app's fetches work. This page is what they see after accepting.
|
|
324
311
|
const CERT_LANDING_HTML = `<!doctype html><html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>sliftutils file server</title></head><body style="font-family:system-ui,sans-serif;max-width:34em;margin:3em auto;padding:0 1em;line-height:1.5;color:#222"><h2>✓ Certificate accepted</h2><p>This is your <b>sliftutils file server</b>. Your browser now trusts its self-signed certificate for this address.</p><p>Return to the app and click <b>Retry</b> (or <b>Connect</b>) to finish connecting with your password.</p><p style="color:#888;font-size:.9em">You can close this tab.</p></body></html>`;
|
|
325
312
|
|
|
326
313
|
export type RemoteFileServerOptions = {
|
|
@@ -328,8 +315,7 @@ export type RemoteFileServerOptions = {
|
|
|
328
315
|
port?: number;
|
|
329
316
|
host?: string;
|
|
330
317
|
password?: string;
|
|
331
|
-
// When true, log batched per-(client, file) access totals every 5s. The CLI turns this on; the
|
|
332
|
-
// library (tests) leaves it off.
|
|
318
|
+
// When true, log batched per-(client, file) access totals every 5s. The CLI turns this on; the library (tests) leaves it off.
|
|
333
319
|
logAccess?: boolean;
|
|
334
320
|
};
|
|
335
321
|
export type RemoteFileServerHandle = { port: number; password: string; url: string; close: () => Promise<void> };
|
|
@@ -343,8 +329,7 @@ export function startRemoteFileServer(options: RemoteFileServerOptions): Promise
|
|
|
343
329
|
const normPassword = normalizePassword(password);
|
|
344
330
|
const { key, cert } = getCert();
|
|
345
331
|
|
|
346
|
-
// Batched access log: aggregate per (ip, op, file) so hammering one file logs once per 5s with the
|
|
347
|
-
// total request count and bytes, instead of a line per request.
|
|
332
|
+
// Batched access log: aggregate per (ip, op, file) so hammering one file logs once per 5s with the total request count and bytes, instead of a line per request.
|
|
348
333
|
type Acc = { ip: string; op: string; path: string; count: number; bytes: number };
|
|
349
334
|
const accessLog = new Map<string, Acc>();
|
|
350
335
|
let flushTimer: ReturnType<typeof setInterval> | undefined;
|
|
@@ -361,8 +346,7 @@ export function startRemoteFileServer(options: RemoteFileServerOptions): Promise
|
|
|
361
346
|
if (!accessLog.size) return;
|
|
362
347
|
const rows = [...accessLog.values()].sort((a, b) => b.bytes - a.bytes);
|
|
363
348
|
accessLog.clear();
|
|
364
|
-
// Fixed-width columns first so lines align, then the variable-length path last. count is the
|
|
365
|
-
// number of requests folded into this row (R = requests, not a multiplier).
|
|
349
|
+
// Fixed-width columns first so lines align, then the variable-length path last. count is the number of requests folded into this row (R = requests, not a multiplier).
|
|
366
350
|
const time = new Date().toTimeString().slice(0, 8);
|
|
367
351
|
for (const e of rows) {
|
|
368
352
|
const size = formatBytes(e.bytes).padStart(9);
|
|
@@ -437,8 +421,7 @@ export function startRemoteFileServer(options: RemoteFileServerOptions): Promise
|
|
|
437
421
|
fs.createReadStream(full, { start, end: end - 1 }).on("error", () => res.end()).pipe(res);
|
|
438
422
|
return;
|
|
439
423
|
}
|
|
440
|
-
// Range-capable media endpoint for <video>/<img>/fetch: honors the Range header (206 +
|
|
441
|
-
// Content-Range + Accept-Ranges) so seeking works. Auth came from the ?token= query above.
|
|
424
|
+
// Range-capable media endpoint for <video>/<img>/fetch: honors the Range header (206 + Content-Range + Accept-Ranges) so seeking works. Auth came from the ?token= query above.
|
|
442
425
|
if ((req.method === "GET" || req.method === "HEAD") && op === "/media") {
|
|
443
426
|
let st: fs.Stats;
|
|
444
427
|
try { st = fs.statSync(full); } catch { return sendJson(404, { error: "not found" }); }
|
|
@@ -483,9 +466,7 @@ export function startRemoteFileServer(options: RemoteFileServerOptions): Promise
|
|
|
483
466
|
}
|
|
484
467
|
});
|
|
485
468
|
|
|
486
|
-
// WebSocket transport: same operations as the HTTP handler, but binary-framed and multiplexed over one
|
|
487
|
-
// socket so large concurrent reads don't pay per-request HTTP overhead. Frame: [u32 headerLen LE][header
|
|
488
|
-
// JSON][body bytes]. Request header {id, op, path, start?, end?, password?}; response {id, status, error?}.
|
|
469
|
+
// WebSocket transport: same operations as the HTTP handler, but binary-framed and multiplexed over one socket so large concurrent reads don't pay per-request HTTP overhead. Frame: [u32 headerLen LE][header JSON][body bytes]. Request header {id, op, path, start?, end?, password?}; response {id, status, error?}.
|
|
489
470
|
const EMPTY = Buffer.alloc(0);
|
|
490
471
|
const wss = new WebSocketServer({ server });
|
|
491
472
|
wss.on("connection", (ws: WebSocket, req: IncomingMessage) => {
|
|
@@ -579,13 +560,9 @@ export function startRemoteFileServer(options: RemoteFileServerOptions): Promise
|
|
|
579
560
|
}
|
|
580
561
|
|
|
581
562
|
// ── server-side compaction (--autocompact) ──
|
|
582
|
-
// Remote clients (e.g. a TV) skip compaction by default — it's expensive to read/rewrite whole files over
|
|
583
|
-
// the network. So the host can do it locally instead: load each bulk database's index and run one merge
|
|
584
|
-
// pass, on startup and every 3 hours. Far more efficient (local disk, no network), and the data still gets
|
|
585
|
-
// compacted eventually.
|
|
563
|
+
// Remote clients (e.g. a TV) skip compaction by default — it's expensive to read/rewrite whole files over the network. So the host can do it locally instead: load each bulk database's index and run one merge pass, on startup and every 3 hours. Far more efficient (local disk, no network), and the data still gets compacted eventually.
|
|
586
564
|
const AUTOCOMPACT_INTERVAL_MS = 3 * 60 * 60 * 1000;
|
|
587
|
-
// One reused compactor instance per collection (keyed by baseDir\0name), so we don't leak the per-instance
|
|
588
|
-
// timers/caches by recreating instances each loop.
|
|
565
|
+
// One reused compactor instance per collection (keyed by baseDir\0name), so we don't leak the per-instance timers/caches by recreating instances each loop.
|
|
589
566
|
const compactors = new Map<string, BulkDatabaseBase<{ key: string }>>();
|
|
590
567
|
function getCompactor(baseDir: string, name: string): BulkDatabaseBase<{ key: string }> {
|
|
591
568
|
const key = baseDir + "\0" + name;
|
|
@@ -603,11 +580,9 @@ function getCompactor(baseDir: string, name: string): BulkDatabaseBase<{ key: st
|
|
|
603
580
|
return db;
|
|
604
581
|
}
|
|
605
582
|
|
|
606
|
-
// One full pass: rescan the disk for collections (new ones may have appeared) and run a merge on each,
|
|
607
|
-
// strictly one after another (never in parallel).
|
|
583
|
+
// One full pass: rescan the disk for collections (new ones may have appeared) and run a merge on each, strictly one after another (never in parallel).
|
|
608
584
|
export async function autocompactBulkDatabases(root: string): Promise<void> {
|
|
609
|
-
// Collections live under <baseDir>/bulkDatabases2/<name>/. The app may nest its data under a "data"
|
|
610
|
-
// subfolder (see getFileStorageNested2's heuristic), so look in both <root> and <root>/data.
|
|
585
|
+
// Collections live under <baseDir>/bulkDatabases2/<name>/. The app may nest its data under a "data" subfolder (see getFileStorageNested2's heuristic), so look in both <root> and <root>/data.
|
|
611
586
|
const collections: { baseDir: string; name: string }[] = [];
|
|
612
587
|
for (const baseDir of [root, path.join(root, "data")]) {
|
|
613
588
|
try {
|
|
@@ -641,8 +616,7 @@ export async function autocompactBulkDatabases(root: string): Promise<void> {
|
|
|
641
616
|
console.log(` [autocompact] done all ${total} collection(s) (${Date.now() - passStart}ms)`);
|
|
642
617
|
}
|
|
643
618
|
|
|
644
|
-
// CLI entry (invoked by bin/filehoster.js or `yarn filehoster <folder>`). Starts the server, logs the
|
|
645
|
-
// password + local/public URLs + batched access, and keeps the UPnP port mapping alive.
|
|
619
|
+
// CLI entry (invoked by bin/filehoster.js or `yarn filehoster <folder>`). Starts the server, logs the password + local/public URLs + batched access, and keeps the UPnP port mapping alive.
|
|
646
620
|
export async function runFileHoster(): Promise<void> {
|
|
647
621
|
const args = process.argv.slice(2);
|
|
648
622
|
const root = args.find(a => !a.startsWith("--") && !a.endsWith(".ts") && !a.endsWith(".js"));
|
|
@@ -657,8 +631,7 @@ export async function runFileHoster(): Promise<void> {
|
|
|
657
631
|
let externalIP: string | undefined;
|
|
658
632
|
try { externalIP = (await getExternalIP()).trim(); } catch { /* offline / unreachable */ }
|
|
659
633
|
|
|
660
|
-
// What the user types into the app: just the host, plus ":port" only if it's not the default. No
|
|
661
|
-
// scheme — the app always uses https.
|
|
634
|
+
// What the user types into the app: just the host, plus ":port" only if it's not the default. No scheme — the app always uses https.
|
|
662
635
|
const host = externalIP || "localhost";
|
|
663
636
|
const appAddress = host + (info.port === 8787 ? "" : ":" + info.port);
|
|
664
637
|
const certUrl = "https://" + host + ":" + info.port;
|
|
@@ -679,8 +652,7 @@ export async function runFileHoster(): Promise<void> {
|
|
|
679
652
|
// No UI to protect on a host, so don't space merges out — compact promptly each pass.
|
|
680
653
|
bulkDatabase2Timing.mergeSpacingMs = 0;
|
|
681
654
|
console.log(" [autocompact] compacting bulk databases on startup, then every 3h (serial)\n");
|
|
682
|
-
// Fire-and-forget: runs the first pass now and re-runs every 3h. Not awaited, so it doesn't block
|
|
683
|
-
// startup; each pass rescans the disk for new collections and merges them one at a time.
|
|
655
|
+
// Fire-and-forget: runs the first pass now and re-runs every 3h. Not awaited, so it doesn't block startup; each pass rescans the disk for new collections and merges them one at a time.
|
|
684
656
|
void runInfinitePollCallAtStart(AUTOCOMPACT_INTERVAL_MS, () => autocompactBulkDatabases(root))
|
|
685
657
|
.catch(e => console.error(" [autocompact] poll error:", (e as Error).stack));
|
|
686
658
|
}
|
|
@@ -1,13 +1,9 @@
|
|
|
1
1
|
import { isNode } from "typesafecss";
|
|
2
2
|
import type { DirectoryWrapper, FileWrapper } from "./FileFolderAPI";
|
|
3
3
|
|
|
4
|
-
// A remote server (remoteFileServer.js) exposed as a DirectoryWrapper — the SAME interface as the
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
//
|
|
8
|
-
// Files here are immutable (written once) or append-only, so the range cache (raw compressed bytes, in
|
|
9
|
-
// ~1MB aligned chunks) is always valid — over the network latency dominates, so reading 1MB costs about
|
|
10
|
-
// the same as 64KB and far fewer round trips wins.
|
|
4
|
+
// A remote server (remoteFileServer.js) exposed as a DirectoryWrapper — the SAME interface as the Node.js and File-System-Access handles. So getDirectoryHandle can return one of these and everything downstream (wrapHandle, navigation, BulkDatabase2) works unchanged; nothing else knows it's remote.
|
|
5
|
+
//
|
|
6
|
+
// Files here are immutable (written once) or append-only, so the range cache (raw compressed bytes, in ~1MB aligned chunks) is always valid — over the network latency dominates, so reading 1MB costs about the same as 64KB and far fewer round trips wins.
|
|
11
7
|
|
|
12
8
|
const EMPTY = Buffer.alloc(0);
|
|
13
9
|
const DEFAULT_CHUNK_BYTES = 1024 * 1024;
|
|
@@ -31,15 +27,11 @@ const sleep = (ms: number) => new Promise<void>(r => setTimeout(r, ms));
|
|
|
31
27
|
|
|
32
28
|
type Stat = { size: number; lastModified: number; dir: boolean };
|
|
33
29
|
|
|
34
|
-
// A stat is cached this long, so the burst of size lookups during a single read pass is one request per
|
|
35
|
-
// file, not one per block. Short enough that an append by another writer shows up promptly; our own
|
|
36
|
-
// writes invalidate it immediately.
|
|
30
|
+
// A stat is cached this long, so the burst of size lookups during a single read pass is one request per file, not one per block. Short enough that an append by another writer shows up promptly; our own writes invalidate it immediately.
|
|
37
31
|
const INFO_TTL_MS = 2000;
|
|
38
|
-
// We keep at most this many bytes of read/write requests in flight at once (the rest queue), so a big
|
|
39
|
-
// read doesn't fire hundreds of MB of requests at the server simultaneously.
|
|
32
|
+
// We keep at most this many bytes of read/write requests in flight at once (the rest queue), so a big read doesn't fire hundreds of MB of requests at the server simultaneously.
|
|
40
33
|
const MAX_INFLIGHT_BYTES = 64 * 1024 * 1024;
|
|
41
|
-
// A single read request fetches at most this much; large reads are split into this many concurrent
|
|
42
|
-
// requests (bounded by MAX_INFLIGHT_BYTES).
|
|
34
|
+
// A single read request fetches at most this much; large reads are split into this many concurrent requests (bounded by MAX_INFLIGHT_BYTES).
|
|
43
35
|
const DEFAULT_MAX_FETCH_BYTES = 4 * 1024 * 1024;
|
|
44
36
|
const STATS_LOG_INTERVAL_MS = 10 * 1000;
|
|
45
37
|
const RECV_WINDOW_MS = 60 * 1000;
|
|
@@ -86,9 +78,7 @@ function decodeFrame(buf: Buffer): { header: any; body: Buffer } {
|
|
|
86
78
|
|
|
87
79
|
type QueuedRequest = { id: number; frame: Buffer; bytes: number; resolve: (r: { status: number; body: Buffer }) => void; reject: (e: Error) => void };
|
|
88
80
|
|
|
89
|
-
// One WebSocket to a remote server, multiplexing all ops over it. Requests are queued and only sent
|
|
90
|
-
// while < MAX_INFLIGHT_BYTES are outstanding; every 10s (when there's traffic) it logs the queue depth
|
|
91
|
-
// and throughput.
|
|
81
|
+
// One WebSocket to a remote server, multiplexing all ops over it. Requests are queued and only sent while < MAX_INFLIGHT_BYTES are outstanding; every 10s (when there's traffic) it logs the queue depth and throughput.
|
|
92
82
|
class Connection {
|
|
93
83
|
private cache: RangeCache;
|
|
94
84
|
private infoCache = new Map<string, { stat: Stat | undefined; at: number }>();
|
|
@@ -197,8 +187,7 @@ class Connection {
|
|
|
197
187
|
}
|
|
198
188
|
}
|
|
199
189
|
|
|
200
|
-
// Sends one request over the WebSocket (queued + throttled). `bytes` is the expected payload size,
|
|
201
|
-
// used for the in-flight cap.
|
|
190
|
+
// Sends one request over the WebSocket (queued + throttled). `bytes` is the expected payload size, used for the in-flight cap.
|
|
202
191
|
private async request(op: string, params: Record<string, unknown>, body: Buffer | undefined, bytes: number): Promise<{ status: number; body: Buffer }> {
|
|
203
192
|
if (this.opts.latencyMs) await sleep(this.opts.latencyMs);
|
|
204
193
|
await this.ensureConnected();
|
|
@@ -254,9 +243,7 @@ class Connection {
|
|
|
254
243
|
}
|
|
255
244
|
}
|
|
256
245
|
|
|
257
|
-
// Caches the longest-known prefix of each aligned chunk per path. Valid because files are immutable or
|
|
258
|
-
// append-only (the bytes at an offset never change). It only does range reads — it doesn't care what
|
|
259
|
-
// the files are.
|
|
246
|
+
// Caches the longest-known prefix of each aligned chunk per path. Valid because files are immutable or append-only (the bytes at an offset never change). It only does range reads — it doesn't care what the files are.
|
|
260
247
|
class RangeCache {
|
|
261
248
|
private chunks = new Map<string, Buffer>(); // path + "" + chunkIndex, insertion-ordered (LRU)
|
|
262
249
|
private bytes = 0;
|
|
@@ -337,8 +324,7 @@ const joinPath = (base: string, key: string) => (base ? base + "/" + key : key);
|
|
|
337
324
|
const baseName = (p: string) => p.split("/").filter(Boolean).pop() || "";
|
|
338
325
|
|
|
339
326
|
class RemoteFileWrapper implements FileWrapper {
|
|
340
|
-
// `stat` is supplied when the parent already statted us (avoids a second /info). `createIntent` means
|
|
341
|
-
// this was opened with create:true, so a missing file reads as empty (it'll be created on write).
|
|
327
|
+
// `stat` is supplied when the parent already statted us (avoids a second /info). `createIntent` means this was opened with create:true, so a missing file reads as empty (it'll be created on write).
|
|
342
328
|
constructor(private conn: Connection, private filePath: string, private stat?: Stat, private createIntent = false) { }
|
|
343
329
|
// Mirror the native FileSystemFileHandle shape so code written against it works unchanged.
|
|
344
330
|
readonly kind = "file" as const;
|
|
@@ -359,8 +345,7 @@ class RemoteFileWrapper implements FileWrapper {
|
|
|
359
345
|
};
|
|
360
346
|
}
|
|
361
347
|
async getURL() {
|
|
362
|
-
// HTTPS URL into the server's range-capable /media endpoint. The token rides in the query because a
|
|
363
|
-
// <video>/<img> element can't send an Authorization header.
|
|
348
|
+
// HTTPS URL into the server's range-capable /media endpoint. The token rides in the query because a <video>/<img> element can't send an Authorization header.
|
|
364
349
|
return `${this.conn.url}/media?path=${encodeURIComponent(this.filePath)}&token=${encodeURIComponent(this.conn.password)}`;
|
|
365
350
|
}
|
|
366
351
|
async createWritable(config?: { keepExistingData?: boolean }) {
|
|
@@ -379,8 +364,7 @@ class RemoteFileWrapper implements FileWrapper {
|
|
|
379
364
|
|
|
380
365
|
class RemoteDirectoryWrapper implements DirectoryWrapper {
|
|
381
366
|
constructor(private conn: Connection, private dirPath: string) { }
|
|
382
|
-
// Mirror the native FileSystemDirectoryHandle shape (name/kind/entries) so code written against the
|
|
383
|
-
// native API — e.g. recursive walks using `handle.entries()` — works the same over the network.
|
|
367
|
+
// Mirror the native FileSystemDirectoryHandle shape (name/kind/entries) so code written against the native API — e.g. recursive walks using `handle.entries()` — works the same over the network.
|
|
384
368
|
readonly kind = "directory" as const;
|
|
385
369
|
readonly isRemote = true;
|
|
386
370
|
get name() { return baseName(this.dirPath); }
|
|
@@ -403,8 +387,7 @@ class RemoteDirectoryWrapper implements DirectoryWrapper {
|
|
|
403
387
|
}
|
|
404
388
|
return new RemoteDirectoryWrapper(this.conn, p); // dirs are created lazily on first write
|
|
405
389
|
}
|
|
406
|
-
// Each entry is itself a real handle (a sub-wrapper), so a recursive walk over .entries() keeps
|
|
407
|
-
// working at every level — exactly like the native API.
|
|
390
|
+
// Each entry is itself a real handle (a sub-wrapper), so a recursive walk over .entries() keeps working at every level — exactly like the native API.
|
|
408
391
|
async *[Symbol.asyncIterator](): AsyncIterableIterator<[string, RemoteFileWrapper | RemoteDirectoryWrapper]> {
|
|
409
392
|
const entries = await this.conn.list(this.dirPath);
|
|
410
393
|
for (const e of entries) {
|
|
@@ -428,9 +411,7 @@ export function getRemoteDirectoryHandle(url: string, password: string, options:
|
|
|
428
411
|
|
|
429
412
|
export type RemoteConnectResult = { status: "ok" } | { status: "unauthorized" } | { status: "unreachable"; error: string };
|
|
430
413
|
|
|
431
|
-
// Verifies a server is reachable and the password works — by opening the WebSocket, authenticating, and
|
|
432
|
-
// listing the root. Distinguishes "connected" / "wrong password" / "couldn't reach it" (the last usually
|
|
433
|
-
// meaning the self-signed cert isn't trusted yet in the browser, since the socket just fails to open).
|
|
414
|
+
// Verifies a server is reachable and the password works — by opening the WebSocket, authenticating, and listing the root. Distinguishes "connected" / "wrong password" / "couldn't reach it" (the last usually meaning the self-signed cert isn't trusted yet in the browser, since the socket just fails to open).
|
|
434
415
|
export async function testRemoteConnection(url: string, password: string, options: RemoteOptions = {}): Promise<RemoteConnectResult> {
|
|
435
416
|
const conn = new Connection(url, password, options);
|
|
436
417
|
try {
|
|
@@ -3,7 +3,6 @@
|
|
|
3
3
|
import { IArchives, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus } from "../IArchives";
|
|
4
4
|
export type ArchivesRemoteConfig = {
|
|
5
5
|
url: string;
|
|
6
|
-
accountName?: string;
|
|
7
6
|
waitForAccess?: boolean;
|
|
8
7
|
};
|
|
9
8
|
export declare function parseStorageUrl(url: string): {
|
|
@@ -29,9 +28,7 @@ export declare class ArchivesRemote implements IArchives {
|
|
|
29
28
|
private lastDeniedLog;
|
|
30
29
|
getDebugName(): string;
|
|
31
30
|
isConnected(): boolean;
|
|
32
|
-
ping(): Promise<{
|
|
33
|
-
takeover?: string;
|
|
34
|
-
}>;
|
|
31
|
+
ping(): Promise<{}>;
|
|
35
32
|
private authenticate;
|
|
36
33
|
private callAuthed;
|
|
37
34
|
waitingForAccess(): Promise<{
|
|
@@ -11,10 +11,7 @@ import {
|
|
|
11
11
|
STORAGE_NOT_AUTHENTICATED, STORAGE_ACCESS_DENIED,
|
|
12
12
|
} from "./storageController";
|
|
13
13
|
|
|
14
|
-
// A bucket on our remote storage server (storageServer.ts), used like ArchivesBackblaze. Works in
|
|
15
|
-
// Node.js and the browser. Authenticates with this machine's certs.ts identity; if the account
|
|
16
|
-
// hasn't trusted this machine yet it requests access, and by default waits, logging instructions
|
|
17
|
-
// every minute (calls block until access is granted).
|
|
14
|
+
// A bucket on our remote storage server (storageServer.ts), used like ArchivesBackblaze. Works in Node.js and the browser. Authenticates with this machine's certs.ts identity; if the account hasn't trusted this machine yet it requests access, and by default waits, logging instructions every minute (calls block until access is granted).
|
|
18
15
|
|
|
19
16
|
const ACCESS_RETRY_DELAY = 1000 * 30;
|
|
20
17
|
const LARGE_FILE_PART_SIZE = 8 * 1024 * 1024;
|
|
@@ -23,10 +20,7 @@ export type ArchivesRemoteConfig = {
|
|
|
23
20
|
// The bucket's routing URL, which addresses the server, account, and bucket in one:
|
|
24
21
|
// https://storage.example.com:4444/file/<account>/<bucketName>/storage/storagerouting.json
|
|
25
22
|
url: string;
|
|
26
|
-
//
|
|
27
|
-
accountName?: string;
|
|
28
|
-
// false: access-denied calls throw immediately (the error includes the access page link)
|
|
29
|
-
// instead of requesting access and blocking until it is granted (the default).
|
|
23
|
+
// false: access-denied calls throw immediately (the error includes the access page link) instead of requesting access and blocking until it is granted (the default).
|
|
30
24
|
waitForAccess?: boolean;
|
|
31
25
|
};
|
|
32
26
|
|
|
@@ -40,8 +34,7 @@ export function parseStorageUrl(url: string): { address: string; port: number }
|
|
|
40
34
|
|
|
41
35
|
// Authenticates a connection to a storage server with this machine's certs.ts identity
|
|
42
36
|
export async function authenticateStorage(config: { address: string; port: number; nodeId: string }): Promise<{ machineId: string; ip: string }> {
|
|
43
|
-
// hostServer nodeIds are machine-specific, so connections by domain must target "any server
|
|
44
|
-
// at this address" (which is how browsers always connect)
|
|
37
|
+
// hostServer nodeIds are machine-specific, so connections by domain must target "any server at this address" (which is how browsers always connect)
|
|
45
38
|
SocketFunction.ENABLE_CLIENT_MODE = true;
|
|
46
39
|
let rootDomain = config.address.split(".").slice(-2).join(".");
|
|
47
40
|
await loadIdentityCA(rootDomain);
|
|
@@ -57,13 +50,12 @@ export async function authenticateStorage(config: { address: string; port: numbe
|
|
|
57
50
|
|
|
58
51
|
export class ArchivesRemote implements IArchives {
|
|
59
52
|
constructor(private config: ArchivesRemoteConfig) {
|
|
60
|
-
// hostServer nodeIds are machine-specific, so connections by domain must target "any
|
|
61
|
-
// server at this address" (which is how browsers always connect)
|
|
53
|
+
// hostServer nodeIds are machine-specific, so connections by domain must target "any server at this address" (which is how browsers always connect)
|
|
62
54
|
SocketFunction.ENABLE_CLIENT_MODE = true;
|
|
63
55
|
}
|
|
64
56
|
|
|
65
57
|
private parsed = parseHostedUrl(this.config.url);
|
|
66
|
-
private account = this.
|
|
58
|
+
private account = this.parsed.account;
|
|
67
59
|
private bucketName = this.parsed.bucketName;
|
|
68
60
|
private nodeId = SocketFunction.connect({ address: this.parsed.address, port: this.parsed.port });
|
|
69
61
|
private controller = RemoteStorageController.nodes[this.nodeId];
|
|
@@ -77,7 +69,7 @@ export class ArchivesRemote implements IArchives {
|
|
|
77
69
|
return SocketFunction.isNodeConnected(this.nodeId);
|
|
78
70
|
}
|
|
79
71
|
|
|
80
|
-
public async ping(): Promise<{
|
|
72
|
+
public async ping(): Promise<{}> {
|
|
81
73
|
return await this.controller.ping();
|
|
82
74
|
}
|
|
83
75
|
|
|
@@ -85,8 +77,7 @@ export class ArchivesRemote implements IArchives {
|
|
|
85
77
|
await authenticateStorage({ address: this.parsed.address, port: this.parsed.port, nodeId: this.nodeId });
|
|
86
78
|
}
|
|
87
79
|
|
|
88
|
-
// Runs a call, authenticating (and re-authenticating after reconnects) as needed. Unlike
|
|
89
|
-
// call(), does NOT wait for account access.
|
|
80
|
+
// Runs a call, authenticating (and re-authenticating after reconnects) as needed. Unlike call(), does NOT wait for account access.
|
|
90
81
|
private async callAuthed<T>(fnc: () => Promise<T>): Promise<T> {
|
|
91
82
|
try {
|
|
92
83
|
return await fnc();
|
|
@@ -97,9 +88,7 @@ export class ArchivesRemote implements IArchives {
|
|
|
97
88
|
}
|
|
98
89
|
}
|
|
99
90
|
|
|
100
|
-
// Returns undefined if this machine has access to the account. Otherwise puts in an access
|
|
101
|
-
// request and returns our machineId + ip (so the caller can display them alongside the link,
|
|
102
|
-
// for the approver to match the incoming request) and the link to the grant page.
|
|
91
|
+
// Returns undefined if this machine has access to the account. Otherwise puts in an access request and returns our machineId + ip (so the caller can display them alongside the link, for the approver to match the incoming request) and the link to the grant page.
|
|
103
92
|
public async waitingForAccess(): Promise<{ link: string; machineId: string; ip: string } | undefined> {
|
|
104
93
|
let state = await this.callAuthed(() => this.controller.getAccessState(this.account));
|
|
105
94
|
if (state.hasAccess) return undefined;
|
|
@@ -116,8 +105,7 @@ export class ArchivesRemote implements IArchives {
|
|
|
116
105
|
return !!state.hasAccess;
|
|
117
106
|
}
|
|
118
107
|
|
|
119
|
-
// Registers our access request server-side (so an admin has a requestId to grant) and logs the
|
|
120
|
-
// grant instructions, at most once a minute
|
|
108
|
+
// Registers our access request server-side (so an admin has a requestId to grant) and logs the grant instructions, at most once a minute
|
|
121
109
|
private async registerAccessRequest(): Promise<void> {
|
|
122
110
|
let requested = await this.callAuthed(() => this.controller.requestAccess(this.account));
|
|
123
111
|
if (Date.now() - this.lastDeniedLog > timeInMinute) {
|
|
@@ -126,9 +114,7 @@ export class ArchivesRemote implements IArchives {
|
|
|
126
114
|
}
|
|
127
115
|
}
|
|
128
116
|
|
|
129
|
-
// Runs a call, authenticating (and re-authenticating after reconnects) and waiting for account
|
|
130
|
-
// access as needed. With waitForAccess false, denied calls throw immediately instead - but the
|
|
131
|
-
// access request is still registered (in the background), so the denial is grantable.
|
|
117
|
+
// Runs a call, authenticating (and re-authenticating after reconnects) and waiting for account access as needed. With waitForAccess false, denied calls throw immediately instead - but the access request is still registered (in the background), so the denial is grantable.
|
|
132
118
|
private async call<T>(fnc: () => Promise<T>): Promise<T> {
|
|
133
119
|
while (true) {
|
|
134
120
|
try {
|
|
@@ -188,8 +174,7 @@ export class ArchivesRemote implements IArchives {
|
|
|
188
174
|
}
|
|
189
175
|
|
|
190
176
|
public async setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined> }): Promise<void> {
|
|
191
|
-
// Ensure we're authenticated with access BEFORE consuming any data (the stream cannot be
|
|
192
|
-
// rewound, so we can't use the retry loop around the actual upload)
|
|
177
|
+
// Ensure we're authenticated with access BEFORE consuming any data (the stream cannot be rewound, so we can't use the retry loop around the actual upload)
|
|
193
178
|
await this.call(() => this.controller.getInfo(this.account, this.bucketName, config.path));
|
|
194
179
|
let uploadId = await this.controller.startLargeFile(this.account, this.bucketName, config.path);
|
|
195
180
|
try {
|
|
@@ -5,8 +5,7 @@ import { IArchives, ArchiveFileInfo, ArchivesConfig } from "../IArchives";
|
|
|
5
5
|
import { buildFileUrl } from "./remoteConfig";
|
|
6
6
|
|
|
7
7
|
// Read-only IArchives over a public bucket's plain-URL form (our storage server's
|
|
8
|
-
// /file/<account>/<bucketName>/... route, or a backblaze friendly URL). Used when we have no API
|
|
9
|
-
// access to a source. Only single-file reads work: there is no listing, and writes always throw.
|
|
8
|
+
// /file/<account>/<bucketName>/... route, or a backblaze friendly URL). Used when we have no API access to a source. Only single-file reads work: there is no listing, and writes always throw.
|
|
10
9
|
|
|
11
10
|
export class ArchivesUrl implements IArchives {
|
|
12
11
|
// base is the bucket's public base URL, e.g. https://host:port/file/<account>/<bucketName>
|
|
@@ -47,8 +46,7 @@ export class ArchivesUrl implements IArchives {
|
|
|
47
46
|
if (total && Number.isFinite(total)) {
|
|
48
47
|
size = total;
|
|
49
48
|
}
|
|
50
|
-
// Servers that don't support ranges return the full file with a 200 (ours serves real
|
|
51
|
-
// 206s, but backblaze friendly URLs and proxies may not)
|
|
49
|
+
// Servers that don't support ranges return the full file with a 200 (ours serves real 206s, but backblaze friendly URLs and proxies may not)
|
|
52
50
|
if (range && response.statusCode === 200) {
|
|
53
51
|
data = data.subarray(Math.min(range.start, data.length), Math.min(range.end, data.length));
|
|
54
52
|
}
|
|
@@ -9,11 +9,7 @@ import { SocketFunction } from "socket-function/SocketFunction";
|
|
|
9
9
|
import { RemoteStorageController, AccessState, AccessRequest } from "./storageController";
|
|
10
10
|
import { authenticateStorage } from "./ArchivesRemote";
|
|
11
11
|
|
|
12
|
-
// The storage server's access page. Visit https://<storageDomain>:<port>/<accountName> to request
|
|
13
|
-
// access to that account for this browser's machine identity. Once granted, lists the machines that
|
|
14
|
-
// have access. To approve someone else's request, the user must type in the requester's IP —
|
|
15
|
-
// pending requests are NEVER shown unsolicited (so a trusted user can't accidentally approve a
|
|
16
|
-
// random machine's request).
|
|
12
|
+
// The storage server's access page. Visit https://<storageDomain>:<port>/<accountName> to request access to that account for this browser's machine identity. Once granted, lists the machines that have access. To approve someone else's request, the user must type in the requester's IP — pending requests are NEVER shown unsolicited (so a trusted user can't accidentally approve a random machine's request).
|
|
17
13
|
|
|
18
14
|
const REFRESH_INTERVAL = 1000 * 15;
|
|
19
15
|
const COPIED_RESET_DELAY = 2000;
|
|
@@ -78,6 +78,7 @@ export declare class BlobStore implements IBucketStore {
|
|
|
78
78
|
constructor(folder: string, sources: ArchivesSource[], config?: {
|
|
79
79
|
onIndexChanged?: ((key: string) => void) | undefined;
|
|
80
80
|
readerDiskLimit?: number | undefined;
|
|
81
|
+
onWriteCounted?: ((kind: "original" | "flushed", bytes: number) => void) | undefined;
|
|
81
82
|
} | undefined);
|
|
82
83
|
private stopped;
|
|
83
84
|
private index;
|
|
@@ -102,20 +103,12 @@ export declare class BlobStore implements IBucketStore {
|
|
|
102
103
|
private countEntry;
|
|
103
104
|
private setIndexEntry;
|
|
104
105
|
private deleteIndexEntry;
|
|
105
|
-
/** Applies a config change to the RUNNING store: windows/routes update in place, new sources
|
|
106
|
-
* are added (their sync starts immediately), and removed sources' slots go dead (their scans
|
|
107
|
-
* stop, their index entries drop). The store survives every routine config evolution - it is
|
|
108
|
-
* never destroyed for a source-list change, only for structural flips it cannot express
|
|
109
|
-
* (rawDisk). Pending fast writes are re-capped to the new flush deadline (flushing
|
|
110
|
-
* immediately when it has already passed). */
|
|
106
|
+
/** Applies a config change to the RUNNING store: windows/routes update in place, new sources are added (their sync starts immediately), and removed sources' slots go dead (their scans stop, their index entries drop). The store survives every routine config evolution - it is never destroyed for a source-list change, only for structural flips it cannot express (rawDisk). Pending fast writes are re-capped to the new flush deadline (flushing immediately when it has already passed). */
|
|
111
107
|
updateSources(specs: BlobSourceSpec[]): void;
|
|
112
108
|
private removeSource;
|
|
113
|
-
/** Rescans our own disk's metadata into the index - used around valid window handoffs, where
|
|
114
|
-
* another process wrote files to the shared folder that our index hasn't seen. */
|
|
109
|
+
/** Rescans our own disk's metadata into the index - used around valid window handoffs, where another process wrote files to the shared folder that our index hasn't seen. */
|
|
115
110
|
rescanBase(): Promise<void>;
|
|
116
|
-
/** A boundary scan of the node that owned (part of) our route in the valid window before ours,
|
|
117
|
-
* when that node is different storage (a disk rescan can't see its writes): just its changes
|
|
118
|
-
* since the boundary neighborhood, with matching values pulled onto our own disk. */
|
|
111
|
+
/** A boundary scan of the node that owned (part of) our route in the valid window before ours, when that node is different storage (a disk rescan can't see its writes): just its changes since the boundary neighborhood, with matching values pulled onto our own disk. */
|
|
119
112
|
boundaryScanRemote(source: IArchives, config: {
|
|
120
113
|
since: number;
|
|
121
114
|
route?: [number, number];
|
|
@@ -134,8 +127,7 @@ export declare class BlobStore implements IBucketStore {
|
|
|
134
127
|
readerDiskLimit?: number;
|
|
135
128
|
syncing: SyncActivity[];
|
|
136
129
|
};
|
|
137
|
-
/** Walks the whole index for exact totals - more expensive than getSyncProgress, but immune to
|
|
138
|
-
* any drift in the maintained counters (and loads the index first, so it's never cold zeros). */
|
|
130
|
+
/** Walks the whole index for exact totals - more expensive than getSyncProgress, but immune to any drift in the maintained counters (and loads the index first, so it's never cold zeros). */
|
|
139
131
|
computeIndexTotals(): Promise<{
|
|
140
132
|
fileCount: number;
|
|
141
133
|
byteCount: number;
|