sliftutils 1.7.10 → 1.7.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/examplestorage/exampleserver.ts +0 -2
  2. package/index.d.ts +290 -54
  3. package/misc/dist/environment.ts.cache +3 -3
  4. package/misc/dist/getSecret.ts.cache +139 -0
  5. package/misc/dist/strings.ts.cache +13 -0
  6. package/misc/dist/zip.ts.cache +9 -0
  7. package/misc/getSecret.ts +38 -16
  8. package/misc/https/cloudflareHelpers.d.ts +19 -0
  9. package/misc/https/cloudflareHelpers.ts +69 -0
  10. package/misc/https/dist/certs.ts.cache +115 -179
  11. package/misc/https/dist/cloudflareHelpers.ts.cache +68 -0
  12. package/misc/https/dist/dns.ts.cache +112 -100
  13. package/misc/https/dist/hostServer.ts.cache +130 -0
  14. package/misc/https/dist/httpsCerts.ts.cache +24 -5
  15. package/misc/https/dist/persistentLocalStorage.ts.cache +3 -2
  16. package/misc/https/dns.d.ts +41 -12
  17. package/misc/https/dns.ts +104 -101
  18. package/misc/https/hostServer.d.ts +0 -6
  19. package/misc/https/hostServer.ts +9 -5
  20. package/package.json +2 -2
  21. package/render-utils/dist/observer.tsx.cache +3 -2
  22. package/spec.txt +0 -90
  23. package/storage/ArchivesDisk.d.ts +2 -0
  24. package/storage/ArchivesDisk.ts +7 -3
  25. package/storage/BulkDatabase2/dist/BulkDatabaseBase.ts.cache +1242 -0
  26. package/storage/BulkDatabase2/dist/BulkDatabaseFormat.ts.cache +513 -0
  27. package/storage/BulkDatabase2/dist/BulkDatabaseMerge.ts.cache +352 -0
  28. package/storage/BulkDatabase2/dist/BulkDatabaseReader.ts.cache +306 -0
  29. package/storage/BulkDatabase2/dist/LoadedIndex.ts.cache +435 -0
  30. package/storage/BulkDatabase2/dist/WriteOverlay.ts.cache +58 -0
  31. package/storage/BulkDatabase2/dist/blockCache.ts.cache +179 -0
  32. package/storage/BulkDatabase2/dist/mergeLock.ts.cache +149 -0
  33. package/storage/BulkDatabase2/dist/mergeMarkers.ts.cache +101 -0
  34. package/storage/BulkDatabase2/dist/streamLog.ts.cache +227 -0
  35. package/storage/BulkDatabase2/dist/syncClient.ts.cache +104 -0
  36. package/storage/IArchives.d.ts +47 -15
  37. package/storage/IArchives.ts +86 -31
  38. package/storage/backblaze.d.ts +15 -0
  39. package/storage/backblaze.ts +199 -79
  40. package/storage/dist/ArchivesDisk.ts.cache +373 -0
  41. package/storage/dist/FileFolderAPI.tsx.cache +508 -65
  42. package/storage/dist/IArchives.ts.cache +44 -0
  43. package/storage/dist/IndexedDBFileFolderAPI.ts.cache +8 -2
  44. package/storage/dist/JSONStorage.ts.cache +6 -2
  45. package/storage/dist/PendingManager.tsx.cache +1 -1
  46. package/storage/dist/TransactionStorage.ts.cache +231 -92
  47. package/storage/dist/backblaze.ts.cache +797 -0
  48. package/storage/dist/fileSystemPointer.ts.cache +36 -3
  49. package/storage/dist/remoteFileStorage.ts.cache +495 -0
  50. package/storage/remoteStorage/ArchivesRemote.d.ts +5 -1
  51. package/storage/remoteStorage/ArchivesRemote.ts +28 -8
  52. package/storage/remoteStorage/ArchivesUrl.d.ts +2 -0
  53. package/storage/remoteStorage/ArchivesUrl.ts +16 -4
  54. package/storage/remoteStorage/blobStore.d.ts +52 -3
  55. package/storage/remoteStorage/blobStore.ts +414 -101
  56. package/storage/remoteStorage/createArchives.d.ts +31 -10
  57. package/storage/remoteStorage/createArchives.ts +470 -253
  58. package/storage/remoteStorage/dist/ArchivesRemote.ts.cache +197 -0
  59. package/storage/remoteStorage/dist/ArchivesUrl.ts.cache +85 -0
  60. package/storage/remoteStorage/dist/accessPage.tsx.cache +222 -0
  61. package/storage/remoteStorage/dist/blobStore.ts.cache +837 -0
  62. package/storage/remoteStorage/dist/cliArgs.ts.cache +16 -0
  63. package/storage/remoteStorage/dist/createArchives.ts.cache +610 -0
  64. package/storage/remoteStorage/dist/remoteConfig.ts.cache +206 -0
  65. package/storage/remoteStorage/dist/sourceWrapper.ts.cache +223 -0
  66. package/storage/remoteStorage/dist/storageController.ts.cache +500 -0
  67. package/storage/remoteStorage/dist/storageServer.ts.cache +106 -0
  68. package/storage/remoteStorage/dist/storageServerCli.ts.cache +39 -0
  69. package/storage/remoteStorage/dist/storageServerState.ts.cache +518 -0
  70. package/storage/remoteStorage/remoteConfig.d.ts +8 -0
  71. package/storage/remoteStorage/remoteConfig.ts +105 -5
  72. package/storage/remoteStorage/sourceWrapper.d.ts +41 -0
  73. package/storage/remoteStorage/sourceWrapper.ts +223 -0
  74. package/storage/remoteStorage/spec.md +31 -0
  75. package/storage/remoteStorage/storageController.d.ts +11 -0
  76. package/storage/remoteStorage/storageController.ts +109 -40
  77. package/storage/remoteStorage/storageServer.d.ts +0 -5
  78. package/storage/remoteStorage/storageServer.ts +17 -10
  79. package/storage/remoteStorage/storageServerCli.ts +0 -3
  80. package/storage/remoteStorage/storageServerState.d.ts +7 -1
  81. package/storage/remoteStorage/storageServerState.ts +230 -48
  82. package/testsite/server.ts +0 -2
  83. package/yarn.lock +11 -6
@@ -6,12 +6,14 @@ import { ArchivesDisk } from "../ArchivesDisk";
6
6
  import { BlobStore, IBucketStore } from "./blobStore";
7
7
  import {
8
8
  RemoteConfig, HostedConfig, IArchives, ArchivesSource, ArchiveFileInfo, ArchivesConfig,
9
- ArchivesSyncStatus, DEFAULT_BASE_SYNC_OPTIONS, DEFAULT_SYNC_OPTIONS,
9
+ ArchivesSyncStatus, FULL_VALID_WINDOW,
10
+ WRITE_PAST_WINDOW_GRACE, STORAGE_WRONG_VALID_WINDOW, STORAGE_WRONG_ROUTE, FULL_ROUTE,
10
11
  } from "../IArchives";
11
- import { ROUTING_FILE, parseRoutingData, parseHostedUrl, buildFileUrl } from "./remoteConfig";
12
+ import { ROUTING_FILE, parseRoutingData, parseHostedUrl, buildFileUrl, getConfigVersion, getRoute, routeContains, routeIntersection } from "./remoteConfig";
12
13
  import { createApiArchives } from "./createArchives";
13
14
  import type { IStorage } from "../IStorage";
14
15
  import type { AccessRequest, TrustRecord } from "./storageController";
16
+ import { getArg } from "./cliArgs";
15
17
 
16
18
  // The storage server's global server-side state. hostStorageServer sets the config once at
17
19
  // startup; everything else (system storages, buckets) is a global cache created lazily on first
@@ -69,12 +71,24 @@ export function assertWritesAllowed(): void {
69
71
  // <folder>/system2/<name> — so they share one cache, keyed by name. ("system2" because the layout
70
72
  // changed when bucket configs moved into the buckets themselves; old "system"/"buckets" folders
71
73
  // are simply ignored.)
74
+ // setTrustedMachines is called by consumers BEFORE starting the server, so when the config isn't
75
+ // set yet we fall back to the same --folder arg the CLI starts the server with.
76
+ function getStorageFolder(): string {
77
+ let config = getStorageServerConfigOptional();
78
+ if (config) return config.folder;
79
+ let folder = getArg("folder");
80
+ if (!folder) {
81
+ throw new Error(`Storage server is not initialized and there is no --folder arg, so the storage folder is unknown`);
82
+ }
83
+ return path.resolve(folder);
84
+ }
85
+
72
86
  const systemStorages = new Map<string, Promise<IStorage<unknown>>>();
73
87
  function getSystemStorage<T>(name: string): Promise<IStorage<T>> {
74
88
  let storage = systemStorages.get(name);
75
89
  if (!storage) {
76
90
  storage = (async () => {
77
- let root = await getFileStorageNested2(getStorageServerConfig().folder);
91
+ let root = await getFileStorageNested2(getStorageFolder());
78
92
  let system = await root.folder.getStorage("system2");
79
93
  let transactionName = "storage" + name[0].toUpperCase() + name.slice(1);
80
94
  return new JSONStorage<unknown>(new TransactionStorage(await system.folder.getStorage(name), transactionName));
@@ -90,6 +104,27 @@ export function getRequests(): Promise<IStorage<AccessRequest[]>> {
90
104
  return getSystemStorage<AccessRequest[]>("requests");
91
105
  }
92
106
 
107
+ /** Makes machineIds the complete trust list for the account: machines not in the list lose access, machines already trusted keep their existing record, and missing ones are added. */
108
+ export async function setTrustedMachines(config: { account: string; machineIds: string[] }): Promise<void> {
109
+ let trust = await getTrust();
110
+ let prefix = `${config.account}|`;
111
+ let desired = new Set(config.machineIds);
112
+ for (let key of await trust.getKeys()) {
113
+ if (!key.startsWith(prefix)) continue;
114
+ let machineId = key.slice(prefix.length);
115
+ if (desired.has(machineId)) {
116
+ desired.delete(machineId);
117
+ continue;
118
+ }
119
+ console.log(`Removing trust for machine ${machineId} on account ${config.account}`);
120
+ await trust.remove(key);
121
+ }
122
+ for (let machineId of desired) {
123
+ console.log(`Adding trust for machine ${machineId} on account ${config.account}`);
124
+ await trust.set(`${prefix}${machineId}`, { account: config.account, machineId, ip: "", time: Date.now() });
125
+ }
126
+ }
127
+
93
128
  // ── buckets ──
94
129
 
95
130
  export type LoadedBucket = {
@@ -99,45 +134,133 @@ export type LoadedBucket = {
99
134
  routing: RemoteConfig;
100
135
  // JSON of routing, for change detection
101
136
  routingJSON: string;
102
- // Our own entry in routing.sources (this bucket on this server), holding the bucket options
103
- // (public/fast/rawDisk/immutable/...). undefined when the config doesn't mention us.
137
+ // ALL of our own entries in routing.sources (this bucket on this server). More than one when
138
+ // the bucket's options change over time - one entry per valid window (e.g. immutable dropped,
139
+ // or fast enabled, starting at a scheduled instant).
140
+ selfEntries: HostedConfig[];
141
+ // The entry valid when the store was built - the structural options (rawDisk, which entries
142
+ // are downstream). Per-write options (fast/immutable) are instead evaluated at each write's
143
+ // time, see getWriteConfig/assertMutable. undefined when the config doesn't mention us.
104
144
  self: HostedConfig | undefined;
105
145
  store: IBucketStore;
106
146
  };
107
147
 
108
148
  const buckets = new Map<string, Promise<LoadedBucket | undefined>>();
109
149
 
150
+ // setTimeout cannot represent longer delays; more distant window boundaries re-check after this
151
+ const MAX_REBUILD_TIMER_DELAY = 2 ** 31 - 1;
152
+ // Rebuild slightly after the boundary, so the newly-valid entry is unambiguously active
153
+ const REBUILD_BOUNDARY_BUFFER = 1000;
154
+
110
155
  function getBucketFolder(account: string, bucketName: string): string {
111
156
  return path.join(getStorageServerConfig().folder, "buckets2", account, bucketName);
112
157
  }
113
158
 
114
- function findSelfEntry(routing: RemoteConfig, account: string, bucketName: string): HostedConfig | undefined {
159
+ function findSelfIndexes(routing: RemoteConfig, account: string, bucketName: string): number[] {
115
160
  let { domain, port } = getStorageServerConfig();
116
- for (let source of routing.sources) {
161
+ let indexes: number[] = [];
162
+ for (let i = 0; i < routing.sources.length; i++) {
163
+ let source = routing.sources[i];
117
164
  if (typeof source === "string" || source.type !== "remote") continue;
118
165
  let parsed = parseHostedUrl(source.url);
119
166
  if (parsed.address === domain && parsed.port === port && parsed.account === account && parsed.bucketName === bucketName) {
120
- return source;
167
+ indexes.push(i);
121
168
  }
122
169
  }
123
- return undefined;
170
+ return indexes;
171
+ }
172
+
173
+ // The entry whose valid window contains the time (among the entries covering the key's route,
174
+ // when one is given - a server can hold different shards with different options). Half-open
175
+ // ([start, end)), so clean breaks resolve unambiguously. Falls back to the nearest window when
176
+ // none contains the time - gaps shouldn't exist, but a write must still resolve to SOME
177
+ // configuration.
178
+ function selectEntryAt(entries: HostedConfig[], time: number, route?: number): HostedConfig | undefined {
179
+ if (route !== undefined) {
180
+ let covering = entries.filter(x => routeContains(x.route, route));
181
+ if (covering.length) {
182
+ entries = covering;
183
+ }
184
+ }
185
+ let containing = entries.find(x => x.validWindow[0] <= time && time < x.validWindow[1]);
186
+ if (containing) return containing;
187
+ let best: HostedConfig | undefined;
188
+ let bestDistance = Infinity;
189
+ for (let entry of entries) {
190
+ let distance = Math.min(Math.abs(time - entry.validWindow[0]), Math.abs(time - entry.validWindow[1]));
191
+ if (distance < bestDistance) {
192
+ bestDistance = distance;
193
+ best = entry;
194
+ }
195
+ }
196
+ return best;
197
+ }
198
+
199
+ // Bucket options can change over time (multiple self entries with different valid windows), and
200
+ // the structural ones (rawDisk, the downstream list) are baked into the store - so the store is
201
+ // rebuilt shortly after each of our window boundaries passes, making the newly-valid entry apply.
202
+ function scheduleWindowBoundaryRebuild(loaded: LoadedBucket): void {
203
+ let now = Date.now();
204
+ let boundaries = loaded.selfEntries.flatMap(x => x.validWindow).filter(t => t > now);
205
+ if (!boundaries.length) return;
206
+ let nextBoundary = Math.min(...boundaries);
207
+ let key = `${loaded.account}/${loaded.bucketName}`;
208
+ let timer = setTimeout(() => {
209
+ void (async () => {
210
+ // A routing change may have already replaced this store; only IT gets a boundary timer
211
+ let current = await buckets.get(key);
212
+ if (current !== loaded) return;
213
+ if (Date.now() < nextBoundary) {
214
+ scheduleWindowBoundaryRebuild(loaded);
215
+ return;
216
+ }
217
+ await scheduleRoutingReload(loaded.account, loaded.bucketName, { force: true });
218
+ })();
219
+ }, Math.min(nextBoundary - now + REBUILD_BOUNDARY_BUFFER, MAX_REBUILD_TIMER_DELAY));
220
+ (timer as { unref?: () => void }).unref?.();
124
221
  }
125
222
 
126
223
  function buildBucket(account: string, bucketName: string, routing: RemoteConfig): LoadedBucket {
127
224
  let folder = getBucketFolder(account, bucketName);
128
- let self = findSelfEntry(routing, account, bucketName);
225
+ let selfIndexes = findSelfIndexes(routing, account, bucketName);
226
+ let selfEntries = selfIndexes.map(i => routing.sources[i] as HostedConfig);
227
+ let self = selectEntryAt(selfEntries, Date.now());
228
+ let selfIndex = -1;
229
+ if (self) {
230
+ selfIndex = routing.sources.indexOf(self);
231
+ }
129
232
  let store: IBucketStore;
130
233
  if (self?.rawDisk) {
131
234
  store = new ArchivesDisk(folder);
132
235
  } else {
133
- // Our own disk is the base source; the other routing entries are synchronization sources
236
+ // Our own disk is the base source; only the routing entries DOWNSTREAM from us (after our
237
+ // currently-valid entry) become synchronization sources. Upstream sources sync from us, so
238
+ // writing to or scanning them would just echo our own data back (and make our availability
239
+ // depend on theirs). A server NOT in the list has been removed from the bucket: it keeps
240
+ // its disk data (the config may re-add it), but stops all synchronization - no one
241
+ // contacts a removed source, and scanning/pushing as if we were still one would fight the
242
+ // real chain. The config-level validWindow rides on each ArchivesSource; the disk source
243
+ // shares our currently-valid entry's window (it holds our copy of the data).
244
+ let ownIndexes = new Set(selfIndexes);
245
+ // Our own disk gets no route filter: everything it holds is ours to index and serve
134
246
  let sources: ArchivesSource[] = [{
135
247
  source: new ArchivesDisk(folder),
136
- options: self?.syncOptions || DEFAULT_BASE_SYNC_OPTIONS,
248
+ validWindow: self?.validWindow || FULL_VALID_WINDOW,
137
249
  }];
138
- for (let source of routing.sources) {
139
- if (typeof source === "string" || source === self) continue;
140
- sources.push({ source: createApiArchives(source), options: source.syncOptions || DEFAULT_SYNC_OPTIONS });
250
+ if (selfIndex === -1) {
251
+ console.log(`This server is not in the routing config for bucket ${account}/${bucketName}; keeping its data on disk but no longer synchronizing it`);
252
+ } else {
253
+ for (let i = selfIndex + 1; i < routing.sources.length; i++) {
254
+ let source = routing.sources[i];
255
+ if (typeof source === "string" || ownIndexes.has(i)) continue;
256
+ // Disjoint-shard sources never talk to each other; a partial overlap syncs only
257
+ // the intersection (scans ignore the rest, writes only send matching keys)
258
+ let sharedRoute = routeIntersection(self?.route, source.route);
259
+ if (!sharedRoute) continue;
260
+ // A bounded-cache server (noFullSync on our own entry) must not full-sync from
261
+ // ANY source, or its disk would fill regardless of the limit
262
+ sources.push({ source: createApiArchives(source), validWindow: source.validWindow, route: sharedRoute, noFullSync: source.noFullSync || self?.noFullSync });
263
+ }
141
264
  }
142
265
  store = new BlobStore(folder, sources, {
143
266
  onIndexChanged: key => {
@@ -146,9 +269,12 @@ function buildBucket(account: string, bucketName: string, routing: RemoteConfig)
146
269
  if (key !== ROUTING_FILE) return;
147
270
  void scheduleRoutingReload(account, bucketName);
148
271
  },
272
+ readerDiskLimit: self?.readerDiskLimit,
149
273
  });
150
274
  }
151
- return { account, bucketName, routing, routingJSON: JSON.stringify(routing), self, store };
275
+ let loaded: LoadedBucket = { account, bucketName, routing, routingJSON: JSON.stringify(routing), selfEntries, self, store };
276
+ scheduleWindowBoundaryRebuild(loaded);
277
+ return loaded;
152
278
  }
153
279
 
154
280
  async function loadBucket(account: string, bucketName: string): Promise<LoadedBucket | undefined> {
@@ -181,16 +307,16 @@ export function getLoadedBucket(account: string, bucketName: string): Promise<Lo
181
307
  // Routing reloads are serialized per bucket, so concurrent writes/syncs can't rebuild the same
182
308
  // store twice in parallel
183
309
  const routingReloads = new Map<string, Promise<void>>();
184
- function scheduleRoutingReload(account: string, bucketName: string): Promise<void> {
310
+ function scheduleRoutingReload(account: string, bucketName: string, config?: { force?: boolean }): Promise<void> {
185
311
  let key = `${account}/${bucketName}`;
186
312
  let next = (routingReloads.get(key) || Promise.resolve())
187
- .then(() => checkRoutingChanged(account, bucketName))
313
+ .then(() => checkRoutingChanged(account, bucketName, config?.force))
188
314
  .catch(e => console.error(`Reloading routing config for bucket ${key} failed:`, e));
189
315
  routingReloads.set(key, next);
190
316
  return next;
191
317
  }
192
318
 
193
- async function checkRoutingChanged(account: string, bucketName: string): Promise<void> {
319
+ async function checkRoutingChanged(account: string, bucketName: string, force?: boolean): Promise<void> {
194
320
  let key = `${account}/${bucketName}`;
195
321
  let loaded = await buckets.get(key);
196
322
  if (!loaded) return;
@@ -199,50 +325,95 @@ async function checkRoutingChanged(account: string, bucketName: string): Promise
199
325
  let data = await loaded.store.get(ROUTING_FILE);
200
326
  if (!data) return;
201
327
  let routing = parseRoutingData(data);
202
- if (JSON.stringify(routing) === loaded.routingJSON) return;
203
- console.log(`Routing config changed for bucket ${key}, rebuilding its store`);
328
+ if (!force && JSON.stringify(routing) === loaded.routingJSON) return;
329
+ if (force) {
330
+ console.log(`Rebuilding the store for bucket ${key} (a valid window boundary passed)`);
331
+ } else {
332
+ console.log(`Routing config changed for bucket ${key}, rebuilding its store`);
333
+ }
204
334
  if (loaded.store instanceof BlobStore) {
205
335
  await loaded.store.dispose();
206
336
  }
207
337
  buckets.set(key, Promise.resolve(buildBucket(account, bucketName, routing)));
208
338
  }
209
339
 
210
- function getWriteConfig(bucket: LoadedBucket): { fast?: boolean; writeDelay?: number } {
211
- return { fast: bucket.self?.fast, writeDelay: bucket.self?.writeDelay };
340
+ // Per-write options are evaluated at the WRITE's time (and the key's route), not the current
341
+ // config: the write time is stamped exactly once (at the first node that receives the write) and
342
+ // propagates with the data, so every node resolves the same self entry - and the same options.
343
+ function getWriteConfig(bucket: LoadedBucket, writeTime: number, route: number): { fast?: boolean; writeDelay?: number } {
344
+ let self = selectEntryAt(bucket.selfEntries, writeTime, route);
345
+ return { fast: self?.fast, writeDelay: self?.writeDelay };
212
346
  }
213
347
 
214
- export async function assertMutable(bucket: LoadedBucket, filePath: string): Promise<void> {
215
- if (!bucket.self?.immutable) return;
348
+ export async function assertMutable(bucket: LoadedBucket, filePath: string, writeTime: number): Promise<void> {
349
+ let self = selectEntryAt(bucket.selfEntries, writeTime, getRoute(filePath));
350
+ if (!self?.immutable) return;
216
351
  if (await bucket.store.getInfo(filePath)) {
217
- throw new Error(`Bucket ${bucket.account}/${bucket.bucketName} is immutable and ${JSON.stringify(filePath)} already exists, so it cannot be written to`);
352
+ throw new Error(`Bucket ${bucket.account}/${bucket.bucketName} is immutable (at write time ${writeTime}) and ${JSON.stringify(filePath)} already exists, so it cannot be written to`);
353
+ }
354
+ }
355
+
356
+ // Routing writes are serialized per bucket, so two concurrent creates can't both build a store
357
+ // (the loser's store would leak with its sync loops running forever)
358
+ const routingWrites = new Map<string, Promise<void>>();
359
+
360
+ async function writeRoutingConfig(account: string, bucketName: string, data: Buffer, config?: { lastModified?: number }): Promise<void> {
361
+ let key = `${account}/${bucketName}`;
362
+ // Validates before storing anything, so a bad config can't brick the bucket
363
+ let incoming = parseRoutingData(data);
364
+ let loaded = await getLoadedBucket(account, bucketName);
365
+ if (!loaded) {
366
+ await new ArchivesDisk(getBucketFolder(account, bucketName)).set(ROUTING_FILE, data, { lastModified: config?.lastModified });
367
+ buckets.set(key, Promise.resolve(buildBucket(account, bucketName, incoming)));
368
+ console.log(`Created bucket ${key}`);
369
+ return;
370
+ }
371
+ // A changed config must carry a higher version than the one currently stored — otherwise two
372
+ // clients with different configs at the same version would keep clobbering each other. Same
373
+ // content at any version is a harmless no-op and is allowed through.
374
+ if (JSON.stringify(incoming) !== loaded.routingJSON && getConfigVersion(incoming) <= getConfigVersion(loaded.routing)) {
375
+ throw new Error(`Cannot update routing config for bucket ${key}: the new config differs from the current one but its version (${getConfigVersion(incoming)}) is not greater than the current version (${getConfigVersion(loaded.routing)}). Increment the version to update it. Current: ${loaded.routingJSON}. Attempted: ${JSON.stringify(incoming)}`);
218
376
  }
377
+ // Routing writes bypass fast mode (the config must apply immediately and survive a crash)
378
+ // and ignore immutable (the routing file must always stay updatable). An older lastModified
379
+ // no-ops inside set, in which case the reload below sees no change.
380
+ await loaded.store.set(ROUTING_FILE, data, { lastModified: config?.lastModified });
381
+ await scheduleRoutingReload(account, bucketName);
219
382
  }
220
383
 
221
384
  export async function writeBucketFile(account: string, bucketName: string, filePath: string, data: Buffer, config?: { lastModified?: number }): Promise<void> {
222
385
  assertWritesAllowed();
223
- let loaded = await getLoadedBucket(account, bucketName);
224
386
  if (filePath === ROUTING_FILE) {
225
- // Validates before storing anything, so a bad config can't brick the bucket
226
- parseRoutingData(data);
227
- if (!loaded) {
228
- let key = `${account}/${bucketName}`;
229
- await new ArchivesDisk(getBucketFolder(account, bucketName)).set(ROUTING_FILE, data, { lastModified: config?.lastModified });
230
- buckets.set(key, Promise.resolve(buildBucket(account, bucketName, parseRoutingData(data))));
231
- console.log(`Created bucket ${key}`);
232
- return;
233
- }
234
- // Routing writes bypass fast mode (the config must apply immediately and survive a crash)
235
- // and ignore immutable (the routing file must always stay updatable). An older lastModified
236
- // no-ops inside set, in which case the reload below sees no change.
237
- await loaded.store.set(ROUTING_FILE, data, { lastModified: config?.lastModified });
238
- await scheduleRoutingReload(account, bucketName);
239
- return;
387
+ let key = `${account}/${bucketName}`;
388
+ let run = (routingWrites.get(key) || Promise.resolve()).then(() => writeRoutingConfig(account, bucketName, data, config));
389
+ // The chain must survive a failed write, so the stored link swallows the error (the caller
390
+ // still gets it from run)
391
+ routingWrites.set(key, run.then(() => { }, () => { }));
392
+ return await run;
240
393
  }
394
+ let loaded = await getLoadedBucket(account, bucketName);
241
395
  if (!loaded) {
242
396
  throw new Error(`Bucket ${account}/${bucketName} does not exist. Write its routing config to ${JSON.stringify(ROUTING_FILE)} to create it.`);
243
397
  }
244
- await assertMutable(loaded, filePath);
245
- await loaded.store.set(filePath, data, { ...getWriteConfig(loaded), lastModified: config?.lastModified });
398
+ // The write time is determined exactly once, HERE at the first node that receives the write;
399
+ // it propagates as lastModified, so every node evaluates the bucket options at the same instant
400
+ let writeTime = config?.lastModified || Date.now();
401
+ // Only freshly-stamped writes are checked: a write with an explicit lastModified is
402
+ // synchronization/backfill (old data legitimately lands on sources whose window has moved on),
403
+ // but a fresh write reaching us outside our windows/routes means the client resolved its
404
+ // target from a stale view - it must re-resolve and retry at the correct source.
405
+ let route = getRoute(filePath);
406
+ if (!config?.lastModified && loaded.selfEntries.length) {
407
+ let timeValid = loaded.selfEntries.filter(x => writeTime >= x.validWindow[0] - WRITE_PAST_WINDOW_GRACE && writeTime <= x.validWindow[1] + WRITE_PAST_WINDOW_GRACE);
408
+ if (!timeValid.length) {
409
+ throw new Error(`${STORAGE_WRONG_VALID_WINDOW} This server is not a valid write target at ${writeTime} for bucket ${account}/${bucketName} (our valid windows: ${JSON.stringify(loaded.selfEntries.map(x => x.validWindow))}). Re-resolve the currently valid source and retry.`);
410
+ }
411
+ if (!timeValid.some(x => routeContains(x.route, route))) {
412
+ throw new Error(`${STORAGE_WRONG_ROUTE} This server does not handle route ${route} (key ${JSON.stringify(filePath)}) for bucket ${account}/${bucketName} (our routes at this time: ${JSON.stringify(timeValid.map(x => x.route || FULL_ROUTE))}). Re-resolve the source for this key and retry.`);
413
+ }
414
+ }
415
+ await assertMutable(loaded, filePath, writeTime);
416
+ await loaded.store.set(filePath, data, { ...getWriteConfig(loaded, writeTime, route), lastModified: writeTime });
246
417
  }
247
418
 
248
419
  export async function deleteBucketFile(account: string, bucketName: string, filePath: string): Promise<void> {
@@ -251,7 +422,7 @@ export async function deleteBucketFile(account: string, bucketName: string, file
251
422
  }
252
423
  let loaded = await getLoadedBucket(account, bucketName);
253
424
  if (!loaded) return;
254
- await loaded.store.del(filePath, getWriteConfig(loaded));
425
+ await loaded.store.del(filePath, getWriteConfig(loaded, Date.now(), getRoute(filePath)));
255
426
  }
256
427
 
257
428
  // ── local IArchives access ──
@@ -274,7 +445,7 @@ class ArchivesLocalBucket implements IArchives {
274
445
  let result = await this.get2(fileName, config);
275
446
  return result && result.data || undefined;
276
447
  }
277
- public async get2(fileName: string, config?: { range?: { start: number; end: number } }): Promise<{ data: Buffer; writeTime: number } | undefined> {
448
+ public async get2(fileName: string, config?: { range?: { start: number; end: number } }): Promise<{ data: Buffer; writeTime: number; size: number } | undefined> {
278
449
  let bucket = await this.getBucket();
279
450
  if (!bucket) return undefined;
280
451
  return await bucket.store.get2(fileName, config);
@@ -291,7 +462,7 @@ class ArchivesLocalBucket implements IArchives {
291
462
  if (!bucket) {
292
463
  throw new Error(`Bucket ${this.account}/${this.bucketName} does not exist. Write its routing config to ${JSON.stringify(ROUTING_FILE)} to create it.`);
293
464
  }
294
- await assertMutable(bucket, config.path);
465
+ await assertMutable(bucket, config.path, Date.now());
295
466
  let id = await bucket.store.startLargeUpload();
296
467
  try {
297
468
  while (true) {
@@ -327,7 +498,18 @@ class ArchivesLocalBucket implements IArchives {
327
498
  public async getConfig(): Promise<ArchivesConfig> {
328
499
  let bucket = await this.getBucket();
329
500
  // Missing buckets say true, matching what they become once created (the default store type)
330
- return { supportsChangesAfter: !bucket || !!bucket.store.getChangesAfter };
501
+ let progress = bucket?.store.getSyncProgress?.();
502
+ return {
503
+ supportsChangesAfter: !bucket || !!bucket.store.getChangesAfter,
504
+ remoteConfig: bucket?.routing,
505
+ index: progress?.index,
506
+ indexSources: progress?.sources,
507
+ readerDiskLimit: progress?.readerDiskLimit,
508
+ syncing: progress?.syncing,
509
+ };
510
+ }
511
+ public async hasWriteAccess(): Promise<boolean> {
512
+ return true;
331
513
  }
332
514
  public async getChangesAfter(time: number): Promise<ArchiveFileInfo[]> {
333
515
  let bucket = await this.getBucket();
@@ -1,5 +1,4 @@
1
1
  process.env.NODE_ENV = "production";
2
- import os from "os";
3
2
  import path from "path";
4
3
  import { SocketFunction } from "socket-function/SocketFunction";
5
4
  import { RequireController } from "socket-function/require/RequireController";
@@ -30,7 +29,6 @@ async function main() {
30
29
  await hostServer({
31
30
  domain: DOMAIN,
32
31
  port: PORT,
33
- cloudflareApiToken: { path: os.homedir() + "/vidgridweb.com.key" },
34
32
  setDNSRecord: true,
35
33
  });
36
34
  }
package/yarn.lock CHANGED
@@ -1944,10 +1944,10 @@ slash@^3.0.0:
1944
1944
  resolved "https://registry.yarnpkg.com/slash/-/slash-3.0.0.tgz#6539be870c165adbd5240220dbe361f1bc4d4634"
1945
1945
  integrity sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==
1946
1946
 
1947
- socket-function@^1.2.4:
1948
- version "1.2.4"
1949
- resolved "https://registry.yarnpkg.com/socket-function/-/socket-function-1.2.4.tgz#3f62a5549c663f032d9cdf21388c74bbeb6e1d91"
1950
- integrity sha512-tKlnF9rK+K9XKWq9SAg4EvyAFbNK74YpjT63F2F8u8XsUDoDDGCSL3H6m2rZnpt0EgpzIXX7sTD9m4OWwMHp2g==
1947
+ socket-function@^1.2.16:
1948
+ version "1.2.16"
1949
+ resolved "https://registry.yarnpkg.com/socket-function/-/socket-function-1.2.16.tgz#6ad7fc1971202d9768582b0212f086675fba61b5"
1950
+ integrity sha512-o3GATr5Fa+496StsPWJIb0yytY/XRMZjJQ4cgb1LOQIfFv7uJUJ3NZtcldMeN/kOmqI0hBaMvhRLC3VHke9awA==
1951
1951
  dependencies:
1952
1952
  "@types/pako" "^2.0.3"
1953
1953
  "@types/ws" "^8.5.3"
@@ -1957,7 +1957,7 @@ socket-function@^1.2.4:
1957
1957
  preact "10.24.3"
1958
1958
  typedev "^0.4.0"
1959
1959
  typenode "^6.0.1"
1960
- ws "^8.17.1"
1960
+ ws "^8.21.0"
1961
1961
  optionalDependencies:
1962
1962
  lmdb "^3.5.1"
1963
1963
 
@@ -2177,11 +2177,16 @@ wrappy@1:
2177
2177
  resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"
2178
2178
  integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==
2179
2179
 
2180
- ws@^8.17.1, ws@^8.18.3:
2180
+ ws@^8.18.3:
2181
2181
  version "8.18.3"
2182
2182
  resolved "https://registry.yarnpkg.com/ws/-/ws-8.18.3.tgz#b56b88abffde62791c639170400c93dcb0c95472"
2183
2183
  integrity sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==
2184
2184
 
2185
+ ws@^8.21.0:
2186
+ version "8.21.1"
2187
+ resolved "https://registry.yarnpkg.com/ws/-/ws-8.21.1.tgz#045650cd4b1207809e7547146223c3814a9af586"
2188
+ integrity sha512-+0NTnW77fFN/DjQi6k/Sq/Yvk4Sgajw7urW8V+asjXnRgDs9gyGkdb7EzgfhA4goXsRIZKE28fzIXBHEzhuiWw==
2189
+
2185
2190
  y18n@^4.0.0:
2186
2191
  version "4.0.3"
2187
2192
  resolved "https://registry.yarnpkg.com/y18n/-/y18n-4.0.3.tgz#b5f259c82cd6e336921efd7bfd8bf560de9eeedf"