sliftutils 1.7.32 → 1.7.33

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. package/builders/extensionBuild.ts +1 -1
  2. package/bundler/bundleEntry.ts +1 -3
  3. package/bundler/bundler.ts +2 -4
  4. package/bundler/sourceMaps.ts +1 -2
  5. package/examplestorage/browser.tsx +1 -2
  6. package/examplestorage/exampleserver.ts +1 -2
  7. package/index.d.ts +105 -100
  8. package/misc/dist/environment.ts.cache +2 -2
  9. package/misc/dist/getSecret.ts.cache +2 -2
  10. package/misc/dist/strings.ts.cache +2 -2
  11. package/misc/dist/zip.ts.cache +2 -2
  12. package/misc/https/dist/certs.ts.cache +2 -2
  13. package/misc/https/dist/dns.ts.cache +7 -16
  14. package/misc/https/dist/persistentLocalStorage.ts.cache +2 -2
  15. package/misc/https/dns.d.ts +1 -4
  16. package/misc/https/dns.ts +5 -14
  17. package/misc/https/hostServer.d.ts +2 -0
  18. package/misc/https/hostServer.ts +6 -7
  19. package/misc/https/httpsCerts.ts +4 -10
  20. package/misc/openrouter.ts +1 -2
  21. package/package.json +1 -1
  22. package/render-utils/FullscreenModal.tsx +1 -4
  23. package/render-utils/GenericFormat.tsx +2 -8
  24. package/render-utils/Input.tsx +5 -13
  25. package/render-utils/InputLabel.tsx +4 -6
  26. package/render-utils/InputPicker.tsx +1 -2
  27. package/render-utils/URLParam.ts +2 -3
  28. package/render-utils/dist/observer.tsx.cache +2 -2
  29. package/render-utils/mobxTyped.ts +2 -3
  30. package/render-utils/niceStringify.ts +2 -4
  31. package/storage/ArchivesDisk.ts +4 -14
  32. package/storage/BulkDatabase2/BulkDatabase2.d.ts +1 -2
  33. package/storage/BulkDatabase2/BulkDatabase2.ts +4 -11
  34. package/storage/BulkDatabase2/BulkDatabaseBase.ts +42 -114
  35. package/storage/BulkDatabase2/BulkDatabaseFormat.ts +31 -91
  36. package/storage/BulkDatabase2/BulkDatabaseMerge.ts +19 -57
  37. package/storage/BulkDatabase2/BulkDatabaseReader.ts +4 -12
  38. package/storage/BulkDatabase2/LoadedIndex.ts +6 -18
  39. package/storage/BulkDatabase2/WriteOverlay.ts +3 -8
  40. package/storage/BulkDatabase2/blockCache.ts +12 -28
  41. package/storage/BulkDatabase2/dist/BulkDatabaseBase.ts.cache +40 -103
  42. package/storage/BulkDatabase2/dist/BulkDatabaseFormat.ts.cache +19 -50
  43. package/storage/BulkDatabase2/dist/BulkDatabaseMerge.ts.cache +18 -50
  44. package/storage/BulkDatabase2/dist/BulkDatabaseReader.ts.cache +7 -15
  45. package/storage/BulkDatabase2/dist/LoadedIndex.ts.cache +9 -21
  46. package/storage/BulkDatabase2/dist/WriteOverlay.ts.cache +6 -11
  47. package/storage/BulkDatabase2/dist/blockCache.ts.cache +15 -31
  48. package/storage/BulkDatabase2/dist/mergeLock.ts.cache +10 -29
  49. package/storage/BulkDatabase2/dist/mergeMarkers.ts.cache +10 -18
  50. package/storage/BulkDatabase2/dist/streamLog.ts.cache +17 -43
  51. package/storage/BulkDatabase2/dist/syncClient.ts.cache +11 -27
  52. package/storage/BulkDatabase2/mergeLock.ts +7 -26
  53. package/storage/BulkDatabase2/mergeMarkers.ts +7 -15
  54. package/storage/BulkDatabase2/streamLog.ts +14 -40
  55. package/storage/BulkDatabase2/syncClient.ts +8 -24
  56. package/storage/FileFolderAPI.tsx +28 -75
  57. package/storage/IArchives.d.ts +3 -3
  58. package/storage/IArchives.ts +21 -50
  59. package/storage/IStorage.ts +2 -4
  60. package/storage/PendingManager.tsx +2 -3
  61. package/storage/TransactionStorage.ts +11 -24
  62. package/storage/backblaze.ts +12 -43
  63. package/storage/dist/ArchivesDisk.ts.cache +7 -17
  64. package/storage/dist/FileFolderAPI.tsx.cache +25 -63
  65. package/storage/dist/IArchives.ts.cache +10 -20
  66. package/storage/dist/IndexedDBFileFolderAPI.ts.cache +2 -2
  67. package/storage/dist/JSONStorage.ts.cache +2 -2
  68. package/storage/dist/PendingManager.tsx.cache +5 -6
  69. package/storage/dist/TransactionStorage.ts.cache +14 -27
  70. package/storage/dist/backblaze.ts.cache +15 -46
  71. package/storage/dist/embeddingFormats.ts.cache +11 -24
  72. package/storage/dist/fileSystemPointer.ts.cache +4 -7
  73. package/storage/dist/remoteFileStorage.ts.cache +17 -36
  74. package/storage/embeddingFormats.ts +11 -27
  75. package/storage/fileSystemPointer.ts +2 -7
  76. package/storage/proxydatabase/dist/ivfEmbeddingDatabase.ts.cache +8 -18
  77. package/storage/proxydatabase/dist/transactionSet.ts.cache +4 -7
  78. package/storage/proxydatabase/inMemoryDatabase.ts +1 -3
  79. package/storage/proxydatabase/ivfEmbeddingDatabase.ts +8 -23
  80. package/storage/proxydatabase/transactionSet.ts +2 -5
  81. package/storage/remoteFileServer.ts +20 -48
  82. package/storage/remoteFileStorage.ts +14 -33
  83. package/storage/remoteStorage/ArchivesRemote.d.ts +1 -4
  84. package/storage/remoteStorage/ArchivesRemote.ts +11 -26
  85. package/storage/remoteStorage/ArchivesUrl.ts +2 -4
  86. package/storage/remoteStorage/accessPage.tsx +1 -5
  87. package/storage/remoteStorage/blobStore.d.ts +5 -13
  88. package/storage/remoteStorage/blobStore.ts +52 -130
  89. package/storage/remoteStorage/createArchives.d.ts +9 -13
  90. package/storage/remoteStorage/createArchives.ts +43 -187
  91. package/storage/remoteStorage/deployTakeover.d.ts +18 -32
  92. package/storage/remoteStorage/deployTakeover.ts +70 -384
  93. package/storage/remoteStorage/dist/ArchivesRemote.ts.cache +12 -24
  94. package/storage/remoteStorage/dist/ArchivesUrl.ts.cache +5 -7
  95. package/storage/remoteStorage/dist/blobStore.ts.cache +47 -117
  96. package/storage/remoteStorage/dist/cliArgs.ts.cache +2 -2
  97. package/storage/remoteStorage/dist/createArchives.ts.cache +48 -189
  98. package/storage/remoteStorage/dist/deployTakeover.ts.cache +81 -387
  99. package/storage/remoteStorage/dist/intermediateSources.ts.cache +119 -0
  100. package/storage/remoteStorage/dist/remoteConfig.ts.cache +36 -24
  101. package/storage/remoteStorage/dist/sourceWrapper.ts.cache +20 -40
  102. package/storage/remoteStorage/dist/storageClientController.ts.cache +4 -6
  103. package/storage/remoteStorage/dist/storageController.ts.cache +11 -73
  104. package/storage/remoteStorage/dist/storageServerState.ts.cache +265 -178
  105. package/storage/remoteStorage/grantAccessCli.ts +1 -3
  106. package/storage/remoteStorage/intermediateSources.d.ts +21 -0
  107. package/storage/remoteStorage/intermediateSources.ts +117 -0
  108. package/storage/remoteStorage/remoteConfig.ts +33 -20
  109. package/storage/remoteStorage/sourceWrapper.d.ts +8 -19
  110. package/storage/remoteStorage/sourceWrapper.ts +19 -41
  111. package/storage/remoteStorage/storageClientController.ts +1 -3
  112. package/storage/remoteStorage/storageController.d.ts +4 -5
  113. package/storage/remoteStorage/storageController.ts +10 -81
  114. package/storage/remoteStorage/storageServer.ts +20 -34
  115. package/storage/remoteStorage/storageServerState.d.ts +28 -5
  116. package/storage/remoteStorage/storageServerState.ts +271 -207
@@ -19,13 +19,9 @@ process.on("unhandledRejection", reason => {
19
19
  console.error(`[writeServer] unhandledRejection (continuing):`, (reason as Error)?.stack ?? reason);
20
20
  });
21
21
 
22
- // Remote file server for sliftutils getRemoteFileStorage / BulkDatabase2. Serves one folder on disk over
23
- // self-signed HTTPS, authenticated with an auto-generated 6-word password (sent as
24
- // `Authorization: Bearer <password>`). Run it with `yarn filehoster <folder>` (or the `filehoster` bin).
25
- //
26
- // SECURITY: the cert is self-signed, so a browser must accept it once (open the printed https URL and
27
- // click through), and the Node client connects with cert verification disabled. The password is what
28
- // authorizes access; keep it secret.
22
+ // Remote file server for sliftutils getRemoteFileStorage / BulkDatabase2. Serves one folder on disk over self-signed HTTPS, authenticated with an auto-generated 6-word password (sent as `Authorization: Bearer <password>`). Run it with `yarn filehoster <folder>` (or the `filehoster` bin).
23
+ //
24
+ // SECURITY: the cert is self-signed, so a browser must accept it once (open the printed https URL and click through), and the Node client connects with cert verification disabled. The password is what authorizes access; keep it secret.
29
25
 
30
26
  // Common, memorable words (frequency-ordered subset) for passphrase generation.
31
27
  const WORDS: string[] = [
@@ -207,16 +203,12 @@ export function generatePassword(wordCount: number): string {
207
203
  return words.join(" ");
208
204
  }
209
205
 
210
- // The password is always a list of words, so we compare case-insensitively and ignore any non-letter
211
- // characters. That makes it forgiving of voice dictation and mobile autocorrect (spacing, capitals,
212
- // stray punctuation) without meaningfully weakening a 6-word passphrase.
206
+ // The password is always a list of words, so we compare case-insensitively and ignore any non-letter characters. That makes it forgiving of voice dictation and mobile autocorrect (spacing, capitals, stray punctuation) without meaningfully weakening a 6-word passphrase.
213
207
  function normalizePassword(p: string): string {
214
208
  return String(p || "").toLowerCase().replace(/[^a-z]/g, "");
215
209
  }
216
210
 
217
- // The password is generated once and saved (next to the cert, outside the served folder), then reused
218
- // across restarts — so connected clients keep working after the server bounces, instead of getting a
219
- // new password every time. Override with the PASSWORD env var.
211
+ // The password is generated once and saved (next to the cert, outside the served folder), then reused across restarts — so connected clients keep working after the server bounces, instead of getting a new password every time. Override with the PASSWORD env var.
220
212
  function getOrCreatePassword(): string {
221
213
  const dir = path.join(os.homedir(), ".sliftutils-remote");
222
214
  fs.mkdirSync(dir, { recursive: true });
@@ -230,8 +222,7 @@ function getOrCreatePassword(): string {
230
222
  return pw;
231
223
  }
232
224
 
233
- // Generate (once) and cache a self-signed key + cert via openssl, outside the served folder so its
234
- // private key is never reachable through the API.
225
+ // Generate (once) and cache a self-signed key + cert via openssl, outside the served folder so its private key is never reachable through the API.
235
226
  function getCert(): { key: Buffer; cert: Buffer } {
236
227
  const dir = path.join(os.homedir(), ".sliftutils-remote");
237
228
  fs.mkdirSync(dir, { recursive: true });
@@ -284,9 +275,7 @@ function formatBytes(n: number): string {
284
275
  return (n / 1024 / 1024 / 1024).toFixed(2) + "GB";
285
276
  }
286
277
 
287
- // Parse a single-range HTTP Range header ("bytes=start-end", "bytes=start-", "bytes=-suffix") into an
288
- // INCLUSIVE [start, end]. Returns "unsatisfiable" for a range past the file, or undefined for no/garbled
289
- // range (caller then serves the whole file). Only single ranges are supported (what media players use).
278
+ // Parse a single-range HTTP Range header ("bytes=start-end", "bytes=start-", "bytes=-suffix") into an INCLUSIVE [start, end]. Returns "unsatisfiable" for a range past the file, or undefined for no/garbled range (caller then serves the whole file). Only single ranges are supported (what media players use).
290
279
  function parseRange(header: string, size: number): { start: number; end: number } | "unsatisfiable" | undefined {
291
280
  const m = /^bytes=(\d*)-(\d*)$/.exec(header.trim());
292
281
  if (!m) return undefined;
@@ -318,9 +307,7 @@ function mediaContentType(filePath: string): string {
318
307
  return MEDIA_TYPES[ext] || "application/octet-stream";
319
308
  }
320
309
 
321
- // Public (no-auth) landing page. The browser can't trust a self-signed cert from a background fetch, so
322
- // the user opens this URL once, accepts the browser's security warning, and the cert becomes trusted for
323
- // the origin — then the app's fetches work. This page is what they see after accepting.
310
+ // Public (no-auth) landing page. The browser can't trust a self-signed cert from a background fetch, so the user opens this URL once, accepts the browser's security warning, and the cert becomes trusted for the origin — then the app's fetches work. This page is what they see after accepting.
324
311
  const CERT_LANDING_HTML = `<!doctype html><html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>sliftutils file server</title></head><body style="font-family:system-ui,sans-serif;max-width:34em;margin:3em auto;padding:0 1em;line-height:1.5;color:#222"><h2>&#10003; Certificate accepted</h2><p>This is your <b>sliftutils file server</b>. Your browser now trusts its self-signed certificate for this address.</p><p>Return to the app and click <b>Retry</b> (or <b>Connect</b>) to finish connecting with your password.</p><p style="color:#888;font-size:.9em">You can close this tab.</p></body></html>`;
325
312
 
326
313
  export type RemoteFileServerOptions = {
@@ -328,8 +315,7 @@ export type RemoteFileServerOptions = {
328
315
  port?: number;
329
316
  host?: string;
330
317
  password?: string;
331
- // When true, log batched per-(client, file) access totals every 5s. The CLI turns this on; the
332
- // library (tests) leaves it off.
318
+ // When true, log batched per-(client, file) access totals every 5s. The CLI turns this on; the library (tests) leaves it off.
333
319
  logAccess?: boolean;
334
320
  };
335
321
  export type RemoteFileServerHandle = { port: number; password: string; url: string; close: () => Promise<void> };
@@ -343,8 +329,7 @@ export function startRemoteFileServer(options: RemoteFileServerOptions): Promise
343
329
  const normPassword = normalizePassword(password);
344
330
  const { key, cert } = getCert();
345
331
 
346
- // Batched access log: aggregate per (ip, op, file) so hammering one file logs once per 5s with the
347
- // total request count and bytes, instead of a line per request.
332
+ // Batched access log: aggregate per (ip, op, file) so hammering one file logs once per 5s with the total request count and bytes, instead of a line per request.
348
333
  type Acc = { ip: string; op: string; path: string; count: number; bytes: number };
349
334
  const accessLog = new Map<string, Acc>();
350
335
  let flushTimer: ReturnType<typeof setInterval> | undefined;
@@ -361,8 +346,7 @@ export function startRemoteFileServer(options: RemoteFileServerOptions): Promise
361
346
  if (!accessLog.size) return;
362
347
  const rows = [...accessLog.values()].sort((a, b) => b.bytes - a.bytes);
363
348
  accessLog.clear();
364
- // Fixed-width columns first so lines align, then the variable-length path last. count is the
365
- // number of requests folded into this row (R = requests, not a multiplier).
349
+ // Fixed-width columns first so lines align, then the variable-length path last. count is the number of requests folded into this row (R = requests, not a multiplier).
366
350
  const time = new Date().toTimeString().slice(0, 8);
367
351
  for (const e of rows) {
368
352
  const size = formatBytes(e.bytes).padStart(9);
@@ -437,8 +421,7 @@ export function startRemoteFileServer(options: RemoteFileServerOptions): Promise
437
421
  fs.createReadStream(full, { start, end: end - 1 }).on("error", () => res.end()).pipe(res);
438
422
  return;
439
423
  }
440
- // Range-capable media endpoint for <video>/<img>/fetch: honors the Range header (206 +
441
- // Content-Range + Accept-Ranges) so seeking works. Auth came from the ?token= query above.
424
+ // Range-capable media endpoint for <video>/<img>/fetch: honors the Range header (206 + Content-Range + Accept-Ranges) so seeking works. Auth came from the ?token= query above.
442
425
  if ((req.method === "GET" || req.method === "HEAD") && op === "/media") {
443
426
  let st: fs.Stats;
444
427
  try { st = fs.statSync(full); } catch { return sendJson(404, { error: "not found" }); }
@@ -483,9 +466,7 @@ export function startRemoteFileServer(options: RemoteFileServerOptions): Promise
483
466
  }
484
467
  });
485
468
 
486
- // WebSocket transport: same operations as the HTTP handler, but binary-framed and multiplexed over one
487
- // socket so large concurrent reads don't pay per-request HTTP overhead. Frame: [u32 headerLen LE][header
488
- // JSON][body bytes]. Request header {id, op, path, start?, end?, password?}; response {id, status, error?}.
469
+ // WebSocket transport: same operations as the HTTP handler, but binary-framed and multiplexed over one socket so large concurrent reads don't pay per-request HTTP overhead. Frame: [u32 headerLen LE][header JSON][body bytes]. Request header {id, op, path, start?, end?, password?}; response {id, status, error?}.
489
470
  const EMPTY = Buffer.alloc(0);
490
471
  const wss = new WebSocketServer({ server });
491
472
  wss.on("connection", (ws: WebSocket, req: IncomingMessage) => {
@@ -579,13 +560,9 @@ export function startRemoteFileServer(options: RemoteFileServerOptions): Promise
579
560
  }
580
561
 
581
562
  // ── server-side compaction (--autocompact) ──
582
- // Remote clients (e.g. a TV) skip compaction by default — it's expensive to read/rewrite whole files over
583
- // the network. So the host can do it locally instead: load each bulk database's index and run one merge
584
- // pass, on startup and every 3 hours. Far more efficient (local disk, no network), and the data still gets
585
- // compacted eventually.
563
+ // Remote clients (e.g. a TV) skip compaction by default — it's expensive to read/rewrite whole files over the network. So the host can do it locally instead: load each bulk database's index and run one merge pass, on startup and every 3 hours. Far more efficient (local disk, no network), and the data still gets compacted eventually.
586
564
  const AUTOCOMPACT_INTERVAL_MS = 3 * 60 * 60 * 1000;
587
- // One reused compactor instance per collection (keyed by baseDir\0name), so we don't leak the per-instance
588
- // timers/caches by recreating instances each loop.
565
+ // One reused compactor instance per collection (keyed by baseDir\0name), so we don't leak the per-instance timers/caches by recreating instances each loop.
589
566
  const compactors = new Map<string, BulkDatabaseBase<{ key: string }>>();
590
567
  function getCompactor(baseDir: string, name: string): BulkDatabaseBase<{ key: string }> {
591
568
  const key = baseDir + "\0" + name;
@@ -603,11 +580,9 @@ function getCompactor(baseDir: string, name: string): BulkDatabaseBase<{ key: st
603
580
  return db;
604
581
  }
605
582
 
606
- // One full pass: rescan the disk for collections (new ones may have appeared) and run a merge on each,
607
- // strictly one after another (never in parallel).
583
+ // One full pass: rescan the disk for collections (new ones may have appeared) and run a merge on each, strictly one after another (never in parallel).
608
584
  export async function autocompactBulkDatabases(root: string): Promise<void> {
609
- // Collections live under <baseDir>/bulkDatabases2/<name>/. The app may nest its data under a "data"
610
- // subfolder (see getFileStorageNested2's heuristic), so look in both <root> and <root>/data.
585
+ // Collections live under <baseDir>/bulkDatabases2/<name>/. The app may nest its data under a "data" subfolder (see getFileStorageNested2's heuristic), so look in both <root> and <root>/data.
611
586
  const collections: { baseDir: string; name: string }[] = [];
612
587
  for (const baseDir of [root, path.join(root, "data")]) {
613
588
  try {
@@ -641,8 +616,7 @@ export async function autocompactBulkDatabases(root: string): Promise<void> {
641
616
  console.log(` [autocompact] done all ${total} collection(s) (${Date.now() - passStart}ms)`);
642
617
  }
643
618
 
644
- // CLI entry (invoked by bin/filehoster.js or `yarn filehoster <folder>`). Starts the server, logs the
645
- // password + local/public URLs + batched access, and keeps the UPnP port mapping alive.
619
+ // CLI entry (invoked by bin/filehoster.js or `yarn filehoster <folder>`). Starts the server, logs the password + local/public URLs + batched access, and keeps the UPnP port mapping alive.
646
620
  export async function runFileHoster(): Promise<void> {
647
621
  const args = process.argv.slice(2);
648
622
  const root = args.find(a => !a.startsWith("--") && !a.endsWith(".ts") && !a.endsWith(".js"));
@@ -657,8 +631,7 @@ export async function runFileHoster(): Promise<void> {
657
631
  let externalIP: string | undefined;
658
632
  try { externalIP = (await getExternalIP()).trim(); } catch { /* offline / unreachable */ }
659
633
 
660
- // What the user types into the app: just the host, plus ":port" only if it's not the default. No
661
- // scheme — the app always uses https.
634
+ // What the user types into the app: just the host, plus ":port" only if it's not the default. No scheme — the app always uses https.
662
635
  const host = externalIP || "localhost";
663
636
  const appAddress = host + (info.port === 8787 ? "" : ":" + info.port);
664
637
  const certUrl = "https://" + host + ":" + info.port;
@@ -679,8 +652,7 @@ export async function runFileHoster(): Promise<void> {
679
652
  // No UI to protect on a host, so don't space merges out — compact promptly each pass.
680
653
  bulkDatabase2Timing.mergeSpacingMs = 0;
681
654
  console.log(" [autocompact] compacting bulk databases on startup, then every 3h (serial)\n");
682
- // Fire-and-forget: runs the first pass now and re-runs every 3h. Not awaited, so it doesn't block
683
- // startup; each pass rescans the disk for new collections and merges them one at a time.
655
+ // Fire-and-forget: runs the first pass now and re-runs every 3h. Not awaited, so it doesn't block startup; each pass rescans the disk for new collections and merges them one at a time.
684
656
  void runInfinitePollCallAtStart(AUTOCOMPACT_INTERVAL_MS, () => autocompactBulkDatabases(root))
685
657
  .catch(e => console.error(" [autocompact] poll error:", (e as Error).stack));
686
658
  }
@@ -1,13 +1,9 @@
1
1
  import { isNode } from "typesafecss";
2
2
  import type { DirectoryWrapper, FileWrapper } from "./FileFolderAPI";
3
3
 
4
- // A remote server (remoteFileServer.js) exposed as a DirectoryWrapper — the SAME interface as the
5
- // Node.js and File-System-Access handles. So getDirectoryHandle can return one of these and everything
6
- // downstream (wrapHandle, navigation, BulkDatabase2) works unchanged; nothing else knows it's remote.
7
- //
8
- // Files here are immutable (written once) or append-only, so the range cache (raw compressed bytes, in
9
- // ~1MB aligned chunks) is always valid — over the network latency dominates, so reading 1MB costs about
10
- // the same as 64KB and far fewer round trips wins.
4
+ // A remote server (remoteFileServer.js) exposed as a DirectoryWrapper — the SAME interface as the Node.js and File-System-Access handles. So getDirectoryHandle can return one of these and everything downstream (wrapHandle, navigation, BulkDatabase2) works unchanged; nothing else knows it's remote.
5
+ //
6
+ // Files here are immutable (written once) or append-only, so the range cache (raw compressed bytes, in ~1MB aligned chunks) is always valid over the network latency dominates, so reading 1MB costs about the same as 64KB and far fewer round trips wins.
11
7
 
12
8
  const EMPTY = Buffer.alloc(0);
13
9
  const DEFAULT_CHUNK_BYTES = 1024 * 1024;
@@ -31,15 +27,11 @@ const sleep = (ms: number) => new Promise<void>(r => setTimeout(r, ms));
31
27
 
32
28
  type Stat = { size: number; lastModified: number; dir: boolean };
33
29
 
34
- // A stat is cached this long, so the burst of size lookups during a single read pass is one request per
35
- // file, not one per block. Short enough that an append by another writer shows up promptly; our own
36
- // writes invalidate it immediately.
30
+ // A stat is cached this long, so the burst of size lookups during a single read pass is one request per file, not one per block. Short enough that an append by another writer shows up promptly; our own writes invalidate it immediately.
37
31
  const INFO_TTL_MS = 2000;
38
- // We keep at most this many bytes of read/write requests in flight at once (the rest queue), so a big
39
- // read doesn't fire hundreds of MB of requests at the server simultaneously.
32
+ // We keep at most this many bytes of read/write requests in flight at once (the rest queue), so a big read doesn't fire hundreds of MB of requests at the server simultaneously.
40
33
  const MAX_INFLIGHT_BYTES = 64 * 1024 * 1024;
41
- // A single read request fetches at most this much; large reads are split into this many concurrent
42
- // requests (bounded by MAX_INFLIGHT_BYTES).
34
+ // A single read request fetches at most this much; large reads are split into this many concurrent requests (bounded by MAX_INFLIGHT_BYTES).
43
35
  const DEFAULT_MAX_FETCH_BYTES = 4 * 1024 * 1024;
44
36
  const STATS_LOG_INTERVAL_MS = 10 * 1000;
45
37
  const RECV_WINDOW_MS = 60 * 1000;
@@ -86,9 +78,7 @@ function decodeFrame(buf: Buffer): { header: any; body: Buffer } {
86
78
 
87
79
  type QueuedRequest = { id: number; frame: Buffer; bytes: number; resolve: (r: { status: number; body: Buffer }) => void; reject: (e: Error) => void };
88
80
 
89
- // One WebSocket to a remote server, multiplexing all ops over it. Requests are queued and only sent
90
- // while < MAX_INFLIGHT_BYTES are outstanding; every 10s (when there's traffic) it logs the queue depth
91
- // and throughput.
81
+ // One WebSocket to a remote server, multiplexing all ops over it. Requests are queued and only sent while < MAX_INFLIGHT_BYTES are outstanding; every 10s (when there's traffic) it logs the queue depth and throughput.
92
82
  class Connection {
93
83
  private cache: RangeCache;
94
84
  private infoCache = new Map<string, { stat: Stat | undefined; at: number }>();
@@ -197,8 +187,7 @@ class Connection {
197
187
  }
198
188
  }
199
189
 
200
- // Sends one request over the WebSocket (queued + throttled). `bytes` is the expected payload size,
201
- // used for the in-flight cap.
190
+ // Sends one request over the WebSocket (queued + throttled). `bytes` is the expected payload size, used for the in-flight cap.
202
191
  private async request(op: string, params: Record<string, unknown>, body: Buffer | undefined, bytes: number): Promise<{ status: number; body: Buffer }> {
203
192
  if (this.opts.latencyMs) await sleep(this.opts.latencyMs);
204
193
  await this.ensureConnected();
@@ -254,9 +243,7 @@ class Connection {
254
243
  }
255
244
  }
256
245
 
257
- // Caches the longest-known prefix of each aligned chunk per path. Valid because files are immutable or
258
- // append-only (the bytes at an offset never change). It only does range reads — it doesn't care what
259
- // the files are.
246
+ // Caches the longest-known prefix of each aligned chunk per path. Valid because files are immutable or append-only (the bytes at an offset never change). It only does range reads — it doesn't care what the files are.
260
247
  class RangeCache {
261
248
  private chunks = new Map<string, Buffer>(); // path + "" + chunkIndex, insertion-ordered (LRU)
262
249
  private bytes = 0;
@@ -337,8 +324,7 @@ const joinPath = (base: string, key: string) => (base ? base + "/" + key : key);
337
324
  const baseName = (p: string) => p.split("/").filter(Boolean).pop() || "";
338
325
 
339
326
  class RemoteFileWrapper implements FileWrapper {
340
- // `stat` is supplied when the parent already statted us (avoids a second /info). `createIntent` means
341
- // this was opened with create:true, so a missing file reads as empty (it'll be created on write).
327
+ // `stat` is supplied when the parent already statted us (avoids a second /info). `createIntent` means this was opened with create:true, so a missing file reads as empty (it'll be created on write).
342
328
  constructor(private conn: Connection, private filePath: string, private stat?: Stat, private createIntent = false) { }
343
329
  // Mirror the native FileSystemFileHandle shape so code written against it works unchanged.
344
330
  readonly kind = "file" as const;
@@ -359,8 +345,7 @@ class RemoteFileWrapper implements FileWrapper {
359
345
  };
360
346
  }
361
347
  async getURL() {
362
- // HTTPS URL into the server's range-capable /media endpoint. The token rides in the query because a
363
- // <video>/<img> element can't send an Authorization header.
348
+ // HTTPS URL into the server's range-capable /media endpoint. The token rides in the query because a <video>/<img> element can't send an Authorization header.
364
349
  return `${this.conn.url}/media?path=${encodeURIComponent(this.filePath)}&token=${encodeURIComponent(this.conn.password)}`;
365
350
  }
366
351
  async createWritable(config?: { keepExistingData?: boolean }) {
@@ -379,8 +364,7 @@ class RemoteFileWrapper implements FileWrapper {
379
364
 
380
365
  class RemoteDirectoryWrapper implements DirectoryWrapper {
381
366
  constructor(private conn: Connection, private dirPath: string) { }
382
- // Mirror the native FileSystemDirectoryHandle shape (name/kind/entries) so code written against the
383
- // native API — e.g. recursive walks using `handle.entries()` — works the same over the network.
367
+ // Mirror the native FileSystemDirectoryHandle shape (name/kind/entries) so code written against the native API — e.g. recursive walks using `handle.entries()` — works the same over the network.
384
368
  readonly kind = "directory" as const;
385
369
  readonly isRemote = true;
386
370
  get name() { return baseName(this.dirPath); }
@@ -403,8 +387,7 @@ class RemoteDirectoryWrapper implements DirectoryWrapper {
403
387
  }
404
388
  return new RemoteDirectoryWrapper(this.conn, p); // dirs are created lazily on first write
405
389
  }
406
- // Each entry is itself a real handle (a sub-wrapper), so a recursive walk over .entries() keeps
407
- // working at every level — exactly like the native API.
390
+ // Each entry is itself a real handle (a sub-wrapper), so a recursive walk over .entries() keeps working at every level — exactly like the native API.
408
391
  async *[Symbol.asyncIterator](): AsyncIterableIterator<[string, RemoteFileWrapper | RemoteDirectoryWrapper]> {
409
392
  const entries = await this.conn.list(this.dirPath);
410
393
  for (const e of entries) {
@@ -428,9 +411,7 @@ export function getRemoteDirectoryHandle(url: string, password: string, options:
428
411
 
429
412
  export type RemoteConnectResult = { status: "ok" } | { status: "unauthorized" } | { status: "unreachable"; error: string };
430
413
 
431
- // Verifies a server is reachable and the password works — by opening the WebSocket, authenticating, and
432
- // listing the root. Distinguishes "connected" / "wrong password" / "couldn't reach it" (the last usually
433
- // meaning the self-signed cert isn't trusted yet in the browser, since the socket just fails to open).
414
+ // Verifies a server is reachable and the password works — by opening the WebSocket, authenticating, and listing the root. Distinguishes "connected" / "wrong password" / "couldn't reach it" (the last usually meaning the self-signed cert isn't trusted yet in the browser, since the socket just fails to open).
434
415
  export async function testRemoteConnection(url: string, password: string, options: RemoteOptions = {}): Promise<RemoteConnectResult> {
435
416
  const conn = new Connection(url, password, options);
436
417
  try {
@@ -3,7 +3,6 @@
3
3
  import { IArchives, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus } from "../IArchives";
4
4
  export type ArchivesRemoteConfig = {
5
5
  url: string;
6
- accountName?: string;
7
6
  waitForAccess?: boolean;
8
7
  };
9
8
  export declare function parseStorageUrl(url: string): {
@@ -29,9 +28,7 @@ export declare class ArchivesRemote implements IArchives {
29
28
  private lastDeniedLog;
30
29
  getDebugName(): string;
31
30
  isConnected(): boolean;
32
- ping(): Promise<{
33
- takeover?: string;
34
- }>;
31
+ ping(): Promise<{}>;
35
32
  private authenticate;
36
33
  private callAuthed;
37
34
  waitingForAccess(): Promise<{
@@ -11,10 +11,7 @@ import {
11
11
  STORAGE_NOT_AUTHENTICATED, STORAGE_ACCESS_DENIED,
12
12
  } from "./storageController";
13
13
 
14
- // A bucket on our remote storage server (storageServer.ts), used like ArchivesBackblaze. Works in
15
- // Node.js and the browser. Authenticates with this machine's certs.ts identity; if the account
16
- // hasn't trusted this machine yet it requests access, and by default waits, logging instructions
17
- // every minute (calls block until access is granted).
14
+ // A bucket on our remote storage server (storageServer.ts), used like ArchivesBackblaze. Works in Node.js and the browser. Authenticates with this machine's certs.ts identity; if the account hasn't trusted this machine yet it requests access, and by default waits, logging instructions every minute (calls block until access is granted).
18
15
 
19
16
  const ACCESS_RETRY_DELAY = 1000 * 30;
20
17
  const LARGE_FILE_PART_SIZE = 8 * 1024 * 1024;
@@ -23,10 +20,7 @@ export type ArchivesRemoteConfig = {
23
20
  // The bucket's routing URL, which addresses the server, account, and bucket in one:
24
21
  // https://storage.example.com:4444/file/<account>/<bucketName>/storage/storagerouting.json
25
22
  url: string;
26
- // Access is checked against this account (defaults to the account in the url)
27
- accountName?: string;
28
- // false: access-denied calls throw immediately (the error includes the access page link)
29
- // instead of requesting access and blocking until it is granted (the default).
23
+ // false: access-denied calls throw immediately (the error includes the access page link) instead of requesting access and blocking until it is granted (the default).
30
24
  waitForAccess?: boolean;
31
25
  };
32
26
 
@@ -40,8 +34,7 @@ export function parseStorageUrl(url: string): { address: string; port: number }
40
34
 
41
35
  // Authenticates a connection to a storage server with this machine's certs.ts identity
42
36
  export async function authenticateStorage(config: { address: string; port: number; nodeId: string }): Promise<{ machineId: string; ip: string }> {
43
- // hostServer nodeIds are machine-specific, so connections by domain must target "any server
44
- // at this address" (which is how browsers always connect)
37
+ // hostServer nodeIds are machine-specific, so connections by domain must target "any server at this address" (which is how browsers always connect)
45
38
  SocketFunction.ENABLE_CLIENT_MODE = true;
46
39
  let rootDomain = config.address.split(".").slice(-2).join(".");
47
40
  await loadIdentityCA(rootDomain);
@@ -57,13 +50,12 @@ export async function authenticateStorage(config: { address: string; port: numbe
57
50
 
58
51
  export class ArchivesRemote implements IArchives {
59
52
  constructor(private config: ArchivesRemoteConfig) {
60
- // hostServer nodeIds are machine-specific, so connections by domain must target "any
61
- // server at this address" (which is how browsers always connect)
53
+ // hostServer nodeIds are machine-specific, so connections by domain must target "any server at this address" (which is how browsers always connect)
62
54
  SocketFunction.ENABLE_CLIENT_MODE = true;
63
55
  }
64
56
 
65
57
  private parsed = parseHostedUrl(this.config.url);
66
- private account = this.config.accountName || this.parsed.account;
58
+ private account = this.parsed.account;
67
59
  private bucketName = this.parsed.bucketName;
68
60
  private nodeId = SocketFunction.connect({ address: this.parsed.address, port: this.parsed.port });
69
61
  private controller = RemoteStorageController.nodes[this.nodeId];
@@ -77,7 +69,7 @@ export class ArchivesRemote implements IArchives {
77
69
  return SocketFunction.isNodeConnected(this.nodeId);
78
70
  }
79
71
 
80
- public async ping(): Promise<{ takeover?: string }> {
72
+ public async ping(): Promise<{}> {
81
73
  return await this.controller.ping();
82
74
  }
83
75
 
@@ -85,8 +77,7 @@ export class ArchivesRemote implements IArchives {
85
77
  await authenticateStorage({ address: this.parsed.address, port: this.parsed.port, nodeId: this.nodeId });
86
78
  }
87
79
 
88
- // Runs a call, authenticating (and re-authenticating after reconnects) as needed. Unlike
89
- // call(), does NOT wait for account access.
80
+ // Runs a call, authenticating (and re-authenticating after reconnects) as needed. Unlike call(), does NOT wait for account access.
90
81
  private async callAuthed<T>(fnc: () => Promise<T>): Promise<T> {
91
82
  try {
92
83
  return await fnc();
@@ -97,9 +88,7 @@ export class ArchivesRemote implements IArchives {
97
88
  }
98
89
  }
99
90
 
100
- // Returns undefined if this machine has access to the account. Otherwise puts in an access
101
- // request and returns our machineId + ip (so the caller can display them alongside the link,
102
- // for the approver to match the incoming request) and the link to the grant page.
91
+ // Returns undefined if this machine has access to the account. Otherwise puts in an access request and returns our machineId + ip (so the caller can display them alongside the link, for the approver to match the incoming request) and the link to the grant page.
103
92
  public async waitingForAccess(): Promise<{ link: string; machineId: string; ip: string } | undefined> {
104
93
  let state = await this.callAuthed(() => this.controller.getAccessState(this.account));
105
94
  if (state.hasAccess) return undefined;
@@ -116,8 +105,7 @@ export class ArchivesRemote implements IArchives {
116
105
  return !!state.hasAccess;
117
106
  }
118
107
 
119
- // Registers our access request server-side (so an admin has a requestId to grant) and logs the
120
- // grant instructions, at most once a minute
108
+ // Registers our access request server-side (so an admin has a requestId to grant) and logs the grant instructions, at most once a minute
121
109
  private async registerAccessRequest(): Promise<void> {
122
110
  let requested = await this.callAuthed(() => this.controller.requestAccess(this.account));
123
111
  if (Date.now() - this.lastDeniedLog > timeInMinute) {
@@ -126,9 +114,7 @@ export class ArchivesRemote implements IArchives {
126
114
  }
127
115
  }
128
116
 
129
- // Runs a call, authenticating (and re-authenticating after reconnects) and waiting for account
130
- // access as needed. With waitForAccess false, denied calls throw immediately instead - but the
131
- // access request is still registered (in the background), so the denial is grantable.
117
+ // Runs a call, authenticating (and re-authenticating after reconnects) and waiting for account access as needed. With waitForAccess false, denied calls throw immediately instead - but the access request is still registered (in the background), so the denial is grantable.
132
118
  private async call<T>(fnc: () => Promise<T>): Promise<T> {
133
119
  while (true) {
134
120
  try {
@@ -188,8 +174,7 @@ export class ArchivesRemote implements IArchives {
188
174
  }
189
175
 
190
176
  public async setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined> }): Promise<void> {
191
- // Ensure we're authenticated with access BEFORE consuming any data (the stream cannot be
192
- // rewound, so we can't use the retry loop around the actual upload)
177
+ // Ensure we're authenticated with access BEFORE consuming any data (the stream cannot be rewound, so we can't use the retry loop around the actual upload)
193
178
  await this.call(() => this.controller.getInfo(this.account, this.bucketName, config.path));
194
179
  let uploadId = await this.controller.startLargeFile(this.account, this.bucketName, config.path);
195
180
  try {
@@ -5,8 +5,7 @@ import { IArchives, ArchiveFileInfo, ArchivesConfig } from "../IArchives";
5
5
  import { buildFileUrl } from "./remoteConfig";
6
6
 
7
7
  // Read-only IArchives over a public bucket's plain-URL form (our storage server's
8
- // /file/<account>/<bucketName>/... route, or a backblaze friendly URL). Used when we have no API
9
- // access to a source. Only single-file reads work: there is no listing, and writes always throw.
8
+ // /file/<account>/<bucketName>/... route, or a backblaze friendly URL). Used when we have no API access to a source. Only single-file reads work: there is no listing, and writes always throw.
10
9
 
11
10
  export class ArchivesUrl implements IArchives {
12
11
  // base is the bucket's public base URL, e.g. https://host:port/file/<account>/<bucketName>
@@ -47,8 +46,7 @@ export class ArchivesUrl implements IArchives {
47
46
  if (total && Number.isFinite(total)) {
48
47
  size = total;
49
48
  }
50
- // Servers that don't support ranges return the full file with a 200 (ours serves real
51
- // 206s, but backblaze friendly URLs and proxies may not)
49
+ // Servers that don't support ranges return the full file with a 200 (ours serves real 206s, but backblaze friendly URLs and proxies may not)
52
50
  if (range && response.statusCode === 200) {
53
51
  data = data.subarray(Math.min(range.start, data.length), Math.min(range.end, data.length));
54
52
  }
@@ -9,11 +9,7 @@ import { SocketFunction } from "socket-function/SocketFunction";
9
9
  import { RemoteStorageController, AccessState, AccessRequest } from "./storageController";
10
10
  import { authenticateStorage } from "./ArchivesRemote";
11
11
 
12
- // The storage server's access page. Visit https://<storageDomain>:<port>/<accountName> to request
13
- // access to that account for this browser's machine identity. Once granted, lists the machines that
14
- // have access. To approve someone else's request, the user must type in the requester's IP —
15
- // pending requests are NEVER shown unsolicited (so a trusted user can't accidentally approve a
16
- // random machine's request).
12
+ // The storage server's access page. Visit https://<storageDomain>:<port>/<accountName> to request access to that account for this browser's machine identity. Once granted, lists the machines that have access. To approve someone else's request, the user must type in the requester's IP — pending requests are NEVER shown unsolicited (so a trusted user can't accidentally approve a random machine's request).
17
13
 
18
14
  const REFRESH_INTERVAL = 1000 * 15;
19
15
  const COPIED_RESET_DELAY = 2000;
@@ -78,6 +78,7 @@ export declare class BlobStore implements IBucketStore {
78
78
  constructor(folder: string, sources: ArchivesSource[], config?: {
79
79
  onIndexChanged?: ((key: string) => void) | undefined;
80
80
  readerDiskLimit?: number | undefined;
81
+ onWriteCounted?: ((kind: "original" | "flushed", bytes: number) => void) | undefined;
81
82
  } | undefined);
82
83
  private stopped;
83
84
  private index;
@@ -102,20 +103,12 @@ export declare class BlobStore implements IBucketStore {
102
103
  private countEntry;
103
104
  private setIndexEntry;
104
105
  private deleteIndexEntry;
105
- /** Applies a config change to the RUNNING store: windows/routes update in place, new sources
106
- * are added (their sync starts immediately), and removed sources' slots go dead (their scans
107
- * stop, their index entries drop). The store survives every routine config evolution - it is
108
- * never destroyed for a source-list change, only for structural flips it cannot express
109
- * (rawDisk). Pending fast writes are re-capped to the new flush deadline (flushing
110
- * immediately when it has already passed). */
106
+ /** Applies a config change to the RUNNING store: windows/routes update in place, new sources are added (their sync starts immediately), and removed sources' slots go dead (their scans stop, their index entries drop). The store survives every routine config evolution - it is never destroyed for a source-list change, only for structural flips it cannot express (rawDisk). Pending fast writes are re-capped to the new flush deadline (flushing immediately when it has already passed). */
111
107
  updateSources(specs: BlobSourceSpec[]): void;
112
108
  private removeSource;
113
- /** Rescans our own disk's metadata into the index - used around valid window handoffs, where
114
- * another process wrote files to the shared folder that our index hasn't seen. */
109
+ /** Rescans our own disk's metadata into the index - used around valid window handoffs, where another process wrote files to the shared folder that our index hasn't seen. */
115
110
  rescanBase(): Promise<void>;
116
- /** A boundary scan of the node that owned (part of) our route in the valid window before ours,
117
- * when that node is different storage (a disk rescan can't see its writes): just its changes
118
- * since the boundary neighborhood, with matching values pulled onto our own disk. */
111
+ /** A boundary scan of the node that owned (part of) our route in the valid window before ours, when that node is different storage (a disk rescan can't see its writes): just its changes since the boundary neighborhood, with matching values pulled onto our own disk. */
119
112
  boundaryScanRemote(source: IArchives, config: {
120
113
  since: number;
121
114
  route?: [number, number];
@@ -134,8 +127,7 @@ export declare class BlobStore implements IBucketStore {
134
127
  readerDiskLimit?: number;
135
128
  syncing: SyncActivity[];
136
129
  };
137
- /** Walks the whole index for exact totals - more expensive than getSyncProgress, but immune to
138
- * any drift in the maintained counters (and loads the index first, so it's never cold zeros). */
130
+ /** Walks the whole index for exact totals - more expensive than getSyncProgress, but immune to any drift in the maintained counters (and loads the index first, so it's never cold zeros). */
139
131
  computeIndexTotals(): Promise<{
140
132
  fileCount: number;
141
133
  byteCount: number;