sliftutils 1.7.10 → 1.7.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/settings.local.json +2 -1
- package/examplestorage/exampleserver.ts +0 -2
- package/index.d.ts +301 -54
- package/misc/dist/environment.ts.cache +3 -3
- package/misc/dist/getSecret.ts.cache +139 -0
- package/misc/dist/strings.ts.cache +13 -0
- package/misc/dist/zip.ts.cache +9 -0
- package/misc/getSecret.ts +38 -16
- package/misc/https/cloudflareHelpers.d.ts +19 -0
- package/misc/https/cloudflareHelpers.ts +69 -0
- package/misc/https/dist/certs.ts.cache +115 -179
- package/misc/https/dist/cloudflareHelpers.ts.cache +68 -0
- package/misc/https/dist/dns.ts.cache +112 -100
- package/misc/https/dist/hostServer.ts.cache +130 -0
- package/misc/https/dist/httpsCerts.ts.cache +24 -5
- package/misc/https/dist/persistentLocalStorage.ts.cache +3 -2
- package/misc/https/dns.d.ts +41 -12
- package/misc/https/dns.ts +104 -101
- package/misc/https/hostServer.d.ts +0 -6
- package/misc/https/hostServer.ts +9 -5
- package/package.json +2 -2
- package/render-utils/dist/observer.tsx.cache +3 -2
- package/spec.txt +0 -90
- package/storage/ArchivesDisk.d.ts +2 -0
- package/storage/ArchivesDisk.ts +7 -3
- package/storage/BulkDatabase2/dist/BulkDatabaseBase.ts.cache +1242 -0
- package/storage/BulkDatabase2/dist/BulkDatabaseFormat.ts.cache +513 -0
- package/storage/BulkDatabase2/dist/BulkDatabaseMerge.ts.cache +352 -0
- package/storage/BulkDatabase2/dist/BulkDatabaseReader.ts.cache +306 -0
- package/storage/BulkDatabase2/dist/LoadedIndex.ts.cache +435 -0
- package/storage/BulkDatabase2/dist/WriteOverlay.ts.cache +58 -0
- package/storage/BulkDatabase2/dist/blockCache.ts.cache +179 -0
- package/storage/BulkDatabase2/dist/mergeLock.ts.cache +149 -0
- package/storage/BulkDatabase2/dist/mergeMarkers.ts.cache +101 -0
- package/storage/BulkDatabase2/dist/streamLog.ts.cache +227 -0
- package/storage/BulkDatabase2/dist/syncClient.ts.cache +104 -0
- package/storage/IArchives.d.ts +47 -15
- package/storage/IArchives.ts +86 -31
- package/storage/backblaze.d.ts +15 -0
- package/storage/backblaze.ts +199 -79
- package/storage/dist/ArchivesDisk.ts.cache +373 -0
- package/storage/dist/FileFolderAPI.tsx.cache +508 -65
- package/storage/dist/IArchives.ts.cache +44 -0
- package/storage/dist/IndexedDBFileFolderAPI.ts.cache +8 -2
- package/storage/dist/JSONStorage.ts.cache +6 -2
- package/storage/dist/PendingManager.tsx.cache +1 -1
- package/storage/dist/TransactionStorage.ts.cache +231 -92
- package/storage/dist/backblaze.ts.cache +797 -0
- package/storage/dist/fileSystemPointer.ts.cache +36 -3
- package/storage/dist/remoteFileStorage.ts.cache +495 -0
- package/storage/remoteStorage/ArchivesRemote.d.ts +5 -1
- package/storage/remoteStorage/ArchivesRemote.ts +28 -8
- package/storage/remoteStorage/ArchivesUrl.d.ts +2 -0
- package/storage/remoteStorage/ArchivesUrl.ts +16 -4
- package/storage/remoteStorage/blobStore.d.ts +63 -3
- package/storage/remoteStorage/blobStore.ts +419 -101
- package/storage/remoteStorage/createArchives.d.ts +31 -10
- package/storage/remoteStorage/createArchives.ts +470 -253
- package/storage/remoteStorage/dist/ArchivesRemote.ts.cache +197 -0
- package/storage/remoteStorage/dist/ArchivesUrl.ts.cache +85 -0
- package/storage/remoteStorage/dist/accessPage.tsx.cache +222 -0
- package/storage/remoteStorage/dist/blobStore.ts.cache +837 -0
- package/storage/remoteStorage/dist/cliArgs.ts.cache +16 -0
- package/storage/remoteStorage/dist/createArchives.ts.cache +610 -0
- package/storage/remoteStorage/dist/remoteConfig.ts.cache +206 -0
- package/storage/remoteStorage/dist/sourceWrapper.ts.cache +223 -0
- package/storage/remoteStorage/dist/storageController.ts.cache +500 -0
- package/storage/remoteStorage/dist/storageServer.ts.cache +106 -0
- package/storage/remoteStorage/dist/storageServerCli.ts.cache +39 -0
- package/storage/remoteStorage/dist/storageServerState.ts.cache +518 -0
- package/storage/remoteStorage/remoteConfig.d.ts +8 -0
- package/storage/remoteStorage/remoteConfig.ts +105 -5
- package/storage/remoteStorage/sourceWrapper.d.ts +41 -0
- package/storage/remoteStorage/sourceWrapper.ts +223 -0
- package/storage/remoteStorage/spec.md +31 -0
- package/storage/remoteStorage/storageController.d.ts +11 -0
- package/storage/remoteStorage/storageController.ts +109 -40
- package/storage/remoteStorage/storageServer.d.ts +0 -5
- package/storage/remoteStorage/storageServer.ts +17 -10
- package/storage/remoteStorage/storageServerCli.ts +0 -3
- package/storage/remoteStorage/storageServerState.d.ts +7 -1
- package/storage/remoteStorage/storageServerState.ts +230 -48
- package/testsite/server.ts +0 -2
- package/yarn.lock +11 -6
|
@@ -6,12 +6,14 @@ import { ArchivesDisk } from "../ArchivesDisk";
|
|
|
6
6
|
import { BlobStore, IBucketStore } from "./blobStore";
|
|
7
7
|
import {
|
|
8
8
|
RemoteConfig, HostedConfig, IArchives, ArchivesSource, ArchiveFileInfo, ArchivesConfig,
|
|
9
|
-
ArchivesSyncStatus,
|
|
9
|
+
ArchivesSyncStatus, FULL_VALID_WINDOW,
|
|
10
|
+
WRITE_PAST_WINDOW_GRACE, STORAGE_WRONG_VALID_WINDOW, STORAGE_WRONG_ROUTE, FULL_ROUTE,
|
|
10
11
|
} from "../IArchives";
|
|
11
|
-
import { ROUTING_FILE, parseRoutingData, parseHostedUrl, buildFileUrl } from "./remoteConfig";
|
|
12
|
+
import { ROUTING_FILE, parseRoutingData, parseHostedUrl, buildFileUrl, getConfigVersion, getRoute, routeContains, routeIntersection } from "./remoteConfig";
|
|
12
13
|
import { createApiArchives } from "./createArchives";
|
|
13
14
|
import type { IStorage } from "../IStorage";
|
|
14
15
|
import type { AccessRequest, TrustRecord } from "./storageController";
|
|
16
|
+
import { getArg } from "./cliArgs";
|
|
15
17
|
|
|
16
18
|
// The storage server's global server-side state. hostStorageServer sets the config once at
|
|
17
19
|
// startup; everything else (system storages, buckets) is a global cache created lazily on first
|
|
@@ -69,12 +71,24 @@ export function assertWritesAllowed(): void {
|
|
|
69
71
|
// <folder>/system2/<name> — so they share one cache, keyed by name. ("system2" because the layout
|
|
70
72
|
// changed when bucket configs moved into the buckets themselves; old "system"/"buckets" folders
|
|
71
73
|
// are simply ignored.)
|
|
74
|
+
// setTrustedMachines is called by consumers BEFORE starting the server, so when the config isn't
|
|
75
|
+
// set yet we fall back to the same --folder arg the CLI starts the server with.
|
|
76
|
+
function getStorageFolder(): string {
|
|
77
|
+
let config = getStorageServerConfigOptional();
|
|
78
|
+
if (config) return config.folder;
|
|
79
|
+
let folder = getArg("folder");
|
|
80
|
+
if (!folder) {
|
|
81
|
+
throw new Error(`Storage server is not initialized and there is no --folder arg, so the storage folder is unknown`);
|
|
82
|
+
}
|
|
83
|
+
return path.resolve(folder);
|
|
84
|
+
}
|
|
85
|
+
|
|
72
86
|
const systemStorages = new Map<string, Promise<IStorage<unknown>>>();
|
|
73
87
|
function getSystemStorage<T>(name: string): Promise<IStorage<T>> {
|
|
74
88
|
let storage = systemStorages.get(name);
|
|
75
89
|
if (!storage) {
|
|
76
90
|
storage = (async () => {
|
|
77
|
-
let root = await getFileStorageNested2(
|
|
91
|
+
let root = await getFileStorageNested2(getStorageFolder());
|
|
78
92
|
let system = await root.folder.getStorage("system2");
|
|
79
93
|
let transactionName = "storage" + name[0].toUpperCase() + name.slice(1);
|
|
80
94
|
return new JSONStorage<unknown>(new TransactionStorage(await system.folder.getStorage(name), transactionName));
|
|
@@ -90,6 +104,27 @@ export function getRequests(): Promise<IStorage<AccessRequest[]>> {
|
|
|
90
104
|
return getSystemStorage<AccessRequest[]>("requests");
|
|
91
105
|
}
|
|
92
106
|
|
|
107
|
+
/** Makes machineIds the complete trust list for the account: machines not in the list lose access, machines already trusted keep their existing record, and missing ones are added. */
|
|
108
|
+
export async function setTrustedMachines(config: { account: string; machineIds: string[] }): Promise<void> {
|
|
109
|
+
let trust = await getTrust();
|
|
110
|
+
let prefix = `${config.account}|`;
|
|
111
|
+
let desired = new Set(config.machineIds);
|
|
112
|
+
for (let key of await trust.getKeys()) {
|
|
113
|
+
if (!key.startsWith(prefix)) continue;
|
|
114
|
+
let machineId = key.slice(prefix.length);
|
|
115
|
+
if (desired.has(machineId)) {
|
|
116
|
+
desired.delete(machineId);
|
|
117
|
+
continue;
|
|
118
|
+
}
|
|
119
|
+
console.log(`Removing trust for machine ${machineId} on account ${config.account}`);
|
|
120
|
+
await trust.remove(key);
|
|
121
|
+
}
|
|
122
|
+
for (let machineId of desired) {
|
|
123
|
+
console.log(`Adding trust for machine ${machineId} on account ${config.account}`);
|
|
124
|
+
await trust.set(`${prefix}${machineId}`, { account: config.account, machineId, ip: "", time: Date.now() });
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
93
128
|
// ── buckets ──
|
|
94
129
|
|
|
95
130
|
export type LoadedBucket = {
|
|
@@ -99,45 +134,133 @@ export type LoadedBucket = {
|
|
|
99
134
|
routing: RemoteConfig;
|
|
100
135
|
// JSON of routing, for change detection
|
|
101
136
|
routingJSON: string;
|
|
102
|
-
//
|
|
103
|
-
//
|
|
137
|
+
// ALL of our own entries in routing.sources (this bucket on this server). More than one when
|
|
138
|
+
// the bucket's options change over time - one entry per valid window (e.g. immutable dropped,
|
|
139
|
+
// or fast enabled, starting at a scheduled instant).
|
|
140
|
+
selfEntries: HostedConfig[];
|
|
141
|
+
// The entry valid when the store was built - the structural options (rawDisk, which entries
|
|
142
|
+
// are downstream). Per-write options (fast/immutable) are instead evaluated at each write's
|
|
143
|
+
// time, see getWriteConfig/assertMutable. undefined when the config doesn't mention us.
|
|
104
144
|
self: HostedConfig | undefined;
|
|
105
145
|
store: IBucketStore;
|
|
106
146
|
};
|
|
107
147
|
|
|
108
148
|
const buckets = new Map<string, Promise<LoadedBucket | undefined>>();
|
|
109
149
|
|
|
150
|
+
// setTimeout cannot represent longer delays; more distant window boundaries re-check after this
|
|
151
|
+
const MAX_REBUILD_TIMER_DELAY = 2 ** 31 - 1;
|
|
152
|
+
// Rebuild slightly after the boundary, so the newly-valid entry is unambiguously active
|
|
153
|
+
const REBUILD_BOUNDARY_BUFFER = 1000;
|
|
154
|
+
|
|
110
155
|
function getBucketFolder(account: string, bucketName: string): string {
|
|
111
156
|
return path.join(getStorageServerConfig().folder, "buckets2", account, bucketName);
|
|
112
157
|
}
|
|
113
158
|
|
|
114
|
-
function
|
|
159
|
+
function findSelfIndexes(routing: RemoteConfig, account: string, bucketName: string): number[] {
|
|
115
160
|
let { domain, port } = getStorageServerConfig();
|
|
116
|
-
|
|
161
|
+
let indexes: number[] = [];
|
|
162
|
+
for (let i = 0; i < routing.sources.length; i++) {
|
|
163
|
+
let source = routing.sources[i];
|
|
117
164
|
if (typeof source === "string" || source.type !== "remote") continue;
|
|
118
165
|
let parsed = parseHostedUrl(source.url);
|
|
119
166
|
if (parsed.address === domain && parsed.port === port && parsed.account === account && parsed.bucketName === bucketName) {
|
|
120
|
-
|
|
167
|
+
indexes.push(i);
|
|
121
168
|
}
|
|
122
169
|
}
|
|
123
|
-
return
|
|
170
|
+
return indexes;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
// The entry whose valid window contains the time (among the entries covering the key's route,
|
|
174
|
+
// when one is given - a server can hold different shards with different options). Half-open
|
|
175
|
+
// ([start, end)), so clean breaks resolve unambiguously. Falls back to the nearest window when
|
|
176
|
+
// none contains the time - gaps shouldn't exist, but a write must still resolve to SOME
|
|
177
|
+
// configuration.
|
|
178
|
+
function selectEntryAt(entries: HostedConfig[], time: number, route?: number): HostedConfig | undefined {
|
|
179
|
+
if (route !== undefined) {
|
|
180
|
+
let covering = entries.filter(x => routeContains(x.route, route));
|
|
181
|
+
if (covering.length) {
|
|
182
|
+
entries = covering;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
let containing = entries.find(x => x.validWindow[0] <= time && time < x.validWindow[1]);
|
|
186
|
+
if (containing) return containing;
|
|
187
|
+
let best: HostedConfig | undefined;
|
|
188
|
+
let bestDistance = Infinity;
|
|
189
|
+
for (let entry of entries) {
|
|
190
|
+
let distance = Math.min(Math.abs(time - entry.validWindow[0]), Math.abs(time - entry.validWindow[1]));
|
|
191
|
+
if (distance < bestDistance) {
|
|
192
|
+
bestDistance = distance;
|
|
193
|
+
best = entry;
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
return best;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
// Bucket options can change over time (multiple self entries with different valid windows), and
|
|
200
|
+
// the structural ones (rawDisk, the downstream list) are baked into the store - so the store is
|
|
201
|
+
// rebuilt shortly after each of our window boundaries passes, making the newly-valid entry apply.
|
|
202
|
+
function scheduleWindowBoundaryRebuild(loaded: LoadedBucket): void {
|
|
203
|
+
let now = Date.now();
|
|
204
|
+
let boundaries = loaded.selfEntries.flatMap(x => x.validWindow).filter(t => t > now);
|
|
205
|
+
if (!boundaries.length) return;
|
|
206
|
+
let nextBoundary = Math.min(...boundaries);
|
|
207
|
+
let key = `${loaded.account}/${loaded.bucketName}`;
|
|
208
|
+
let timer = setTimeout(() => {
|
|
209
|
+
void (async () => {
|
|
210
|
+
// A routing change may have already replaced this store; only IT gets a boundary timer
|
|
211
|
+
let current = await buckets.get(key);
|
|
212
|
+
if (current !== loaded) return;
|
|
213
|
+
if (Date.now() < nextBoundary) {
|
|
214
|
+
scheduleWindowBoundaryRebuild(loaded);
|
|
215
|
+
return;
|
|
216
|
+
}
|
|
217
|
+
await scheduleRoutingReload(loaded.account, loaded.bucketName, { force: true });
|
|
218
|
+
})();
|
|
219
|
+
}, Math.min(nextBoundary - now + REBUILD_BOUNDARY_BUFFER, MAX_REBUILD_TIMER_DELAY));
|
|
220
|
+
(timer as { unref?: () => void }).unref?.();
|
|
124
221
|
}
|
|
125
222
|
|
|
126
223
|
function buildBucket(account: string, bucketName: string, routing: RemoteConfig): LoadedBucket {
|
|
127
224
|
let folder = getBucketFolder(account, bucketName);
|
|
128
|
-
let
|
|
225
|
+
let selfIndexes = findSelfIndexes(routing, account, bucketName);
|
|
226
|
+
let selfEntries = selfIndexes.map(i => routing.sources[i] as HostedConfig);
|
|
227
|
+
let self = selectEntryAt(selfEntries, Date.now());
|
|
228
|
+
let selfIndex = -1;
|
|
229
|
+
if (self) {
|
|
230
|
+
selfIndex = routing.sources.indexOf(self);
|
|
231
|
+
}
|
|
129
232
|
let store: IBucketStore;
|
|
130
233
|
if (self?.rawDisk) {
|
|
131
234
|
store = new ArchivesDisk(folder);
|
|
132
235
|
} else {
|
|
133
|
-
// Our own disk is the base source; the
|
|
236
|
+
// Our own disk is the base source; only the routing entries DOWNSTREAM from us (after our
|
|
237
|
+
// currently-valid entry) become synchronization sources. Upstream sources sync from us, so
|
|
238
|
+
// writing to or scanning them would just echo our own data back (and make our availability
|
|
239
|
+
// depend on theirs). A server NOT in the list has been removed from the bucket: it keeps
|
|
240
|
+
// its disk data (the config may re-add it), but stops all synchronization - no one
|
|
241
|
+
// contacts a removed source, and scanning/pushing as if we were still one would fight the
|
|
242
|
+
// real chain. The config-level validWindow rides on each ArchivesSource; the disk source
|
|
243
|
+
// shares our currently-valid entry's window (it holds our copy of the data).
|
|
244
|
+
let ownIndexes = new Set(selfIndexes);
|
|
245
|
+
// Our own disk gets no route filter: everything it holds is ours to index and serve
|
|
134
246
|
let sources: ArchivesSource[] = [{
|
|
135
247
|
source: new ArchivesDisk(folder),
|
|
136
|
-
|
|
248
|
+
validWindow: self?.validWindow || FULL_VALID_WINDOW,
|
|
137
249
|
}];
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
250
|
+
if (selfIndex === -1) {
|
|
251
|
+
console.log(`This server is not in the routing config for bucket ${account}/${bucketName}; keeping its data on disk but no longer synchronizing it`);
|
|
252
|
+
} else {
|
|
253
|
+
for (let i = selfIndex + 1; i < routing.sources.length; i++) {
|
|
254
|
+
let source = routing.sources[i];
|
|
255
|
+
if (typeof source === "string" || ownIndexes.has(i)) continue;
|
|
256
|
+
// Disjoint-shard sources never talk to each other; a partial overlap syncs only
|
|
257
|
+
// the intersection (scans ignore the rest, writes only send matching keys)
|
|
258
|
+
let sharedRoute = routeIntersection(self?.route, source.route);
|
|
259
|
+
if (!sharedRoute) continue;
|
|
260
|
+
// A bounded-cache server (noFullSync on our own entry) must not full-sync from
|
|
261
|
+
// ANY source, or its disk would fill regardless of the limit
|
|
262
|
+
sources.push({ source: createApiArchives(source), validWindow: source.validWindow, route: sharedRoute, noFullSync: source.noFullSync || self?.noFullSync });
|
|
263
|
+
}
|
|
141
264
|
}
|
|
142
265
|
store = new BlobStore(folder, sources, {
|
|
143
266
|
onIndexChanged: key => {
|
|
@@ -146,9 +269,12 @@ function buildBucket(account: string, bucketName: string, routing: RemoteConfig)
|
|
|
146
269
|
if (key !== ROUTING_FILE) return;
|
|
147
270
|
void scheduleRoutingReload(account, bucketName);
|
|
148
271
|
},
|
|
272
|
+
readerDiskLimit: self?.readerDiskLimit,
|
|
149
273
|
});
|
|
150
274
|
}
|
|
151
|
-
|
|
275
|
+
let loaded: LoadedBucket = { account, bucketName, routing, routingJSON: JSON.stringify(routing), selfEntries, self, store };
|
|
276
|
+
scheduleWindowBoundaryRebuild(loaded);
|
|
277
|
+
return loaded;
|
|
152
278
|
}
|
|
153
279
|
|
|
154
280
|
async function loadBucket(account: string, bucketName: string): Promise<LoadedBucket | undefined> {
|
|
@@ -181,16 +307,16 @@ export function getLoadedBucket(account: string, bucketName: string): Promise<Lo
|
|
|
181
307
|
// Routing reloads are serialized per bucket, so concurrent writes/syncs can't rebuild the same
|
|
182
308
|
// store twice in parallel
|
|
183
309
|
const routingReloads = new Map<string, Promise<void>>();
|
|
184
|
-
function scheduleRoutingReload(account: string, bucketName: string): Promise<void> {
|
|
310
|
+
function scheduleRoutingReload(account: string, bucketName: string, config?: { force?: boolean }): Promise<void> {
|
|
185
311
|
let key = `${account}/${bucketName}`;
|
|
186
312
|
let next = (routingReloads.get(key) || Promise.resolve())
|
|
187
|
-
.then(() => checkRoutingChanged(account, bucketName))
|
|
313
|
+
.then(() => checkRoutingChanged(account, bucketName, config?.force))
|
|
188
314
|
.catch(e => console.error(`Reloading routing config for bucket ${key} failed:`, e));
|
|
189
315
|
routingReloads.set(key, next);
|
|
190
316
|
return next;
|
|
191
317
|
}
|
|
192
318
|
|
|
193
|
-
async function checkRoutingChanged(account: string, bucketName: string): Promise<void> {
|
|
319
|
+
async function checkRoutingChanged(account: string, bucketName: string, force?: boolean): Promise<void> {
|
|
194
320
|
let key = `${account}/${bucketName}`;
|
|
195
321
|
let loaded = await buckets.get(key);
|
|
196
322
|
if (!loaded) return;
|
|
@@ -199,50 +325,95 @@ async function checkRoutingChanged(account: string, bucketName: string): Promise
|
|
|
199
325
|
let data = await loaded.store.get(ROUTING_FILE);
|
|
200
326
|
if (!data) return;
|
|
201
327
|
let routing = parseRoutingData(data);
|
|
202
|
-
if (JSON.stringify(routing) === loaded.routingJSON) return;
|
|
203
|
-
|
|
328
|
+
if (!force && JSON.stringify(routing) === loaded.routingJSON) return;
|
|
329
|
+
if (force) {
|
|
330
|
+
console.log(`Rebuilding the store for bucket ${key} (a valid window boundary passed)`);
|
|
331
|
+
} else {
|
|
332
|
+
console.log(`Routing config changed for bucket ${key}, rebuilding its store`);
|
|
333
|
+
}
|
|
204
334
|
if (loaded.store instanceof BlobStore) {
|
|
205
335
|
await loaded.store.dispose();
|
|
206
336
|
}
|
|
207
337
|
buckets.set(key, Promise.resolve(buildBucket(account, bucketName, routing)));
|
|
208
338
|
}
|
|
209
339
|
|
|
210
|
-
|
|
211
|
-
|
|
340
|
+
// Per-write options are evaluated at the WRITE's time (and the key's route), not the current
|
|
341
|
+
// config: the write time is stamped exactly once (at the first node that receives the write) and
|
|
342
|
+
// propagates with the data, so every node resolves the same self entry - and the same options.
|
|
343
|
+
function getWriteConfig(bucket: LoadedBucket, writeTime: number, route: number): { fast?: boolean; writeDelay?: number } {
|
|
344
|
+
let self = selectEntryAt(bucket.selfEntries, writeTime, route);
|
|
345
|
+
return { fast: self?.fast, writeDelay: self?.writeDelay };
|
|
212
346
|
}
|
|
213
347
|
|
|
214
|
-
export async function assertMutable(bucket: LoadedBucket, filePath: string): Promise<void> {
|
|
215
|
-
|
|
348
|
+
export async function assertMutable(bucket: LoadedBucket, filePath: string, writeTime: number): Promise<void> {
|
|
349
|
+
let self = selectEntryAt(bucket.selfEntries, writeTime, getRoute(filePath));
|
|
350
|
+
if (!self?.immutable) return;
|
|
216
351
|
if (await bucket.store.getInfo(filePath)) {
|
|
217
|
-
throw new Error(`Bucket ${bucket.account}/${bucket.bucketName} is immutable and ${JSON.stringify(filePath)} already exists, so it cannot be written to`);
|
|
352
|
+
throw new Error(`Bucket ${bucket.account}/${bucket.bucketName} is immutable (at write time ${writeTime}) and ${JSON.stringify(filePath)} already exists, so it cannot be written to`);
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
// Routing writes are serialized per bucket, so two concurrent creates can't both build a store
|
|
357
|
+
// (the loser's store would leak with its sync loops running forever)
|
|
358
|
+
const routingWrites = new Map<string, Promise<void>>();
|
|
359
|
+
|
|
360
|
+
async function writeRoutingConfig(account: string, bucketName: string, data: Buffer, config?: { lastModified?: number }): Promise<void> {
|
|
361
|
+
let key = `${account}/${bucketName}`;
|
|
362
|
+
// Validates before storing anything, so a bad config can't brick the bucket
|
|
363
|
+
let incoming = parseRoutingData(data);
|
|
364
|
+
let loaded = await getLoadedBucket(account, bucketName);
|
|
365
|
+
if (!loaded) {
|
|
366
|
+
await new ArchivesDisk(getBucketFolder(account, bucketName)).set(ROUTING_FILE, data, { lastModified: config?.lastModified });
|
|
367
|
+
buckets.set(key, Promise.resolve(buildBucket(account, bucketName, incoming)));
|
|
368
|
+
console.log(`Created bucket ${key}`);
|
|
369
|
+
return;
|
|
370
|
+
}
|
|
371
|
+
// A changed config must carry a higher version than the one currently stored — otherwise two
|
|
372
|
+
// clients with different configs at the same version would keep clobbering each other. Same
|
|
373
|
+
// content at any version is a harmless no-op and is allowed through.
|
|
374
|
+
if (JSON.stringify(incoming) !== loaded.routingJSON && getConfigVersion(incoming) <= getConfigVersion(loaded.routing)) {
|
|
375
|
+
throw new Error(`Cannot update routing config for bucket ${key}: the new config differs from the current one but its version (${getConfigVersion(incoming)}) is not greater than the current version (${getConfigVersion(loaded.routing)}). Increment the version to update it. Current: ${loaded.routingJSON}. Attempted: ${JSON.stringify(incoming)}`);
|
|
218
376
|
}
|
|
377
|
+
// Routing writes bypass fast mode (the config must apply immediately and survive a crash)
|
|
378
|
+
// and ignore immutable (the routing file must always stay updatable). An older lastModified
|
|
379
|
+
// no-ops inside set, in which case the reload below sees no change.
|
|
380
|
+
await loaded.store.set(ROUTING_FILE, data, { lastModified: config?.lastModified });
|
|
381
|
+
await scheduleRoutingReload(account, bucketName);
|
|
219
382
|
}
|
|
220
383
|
|
|
221
384
|
export async function writeBucketFile(account: string, bucketName: string, filePath: string, data: Buffer, config?: { lastModified?: number }): Promise<void> {
|
|
222
385
|
assertWritesAllowed();
|
|
223
|
-
let loaded = await getLoadedBucket(account, bucketName);
|
|
224
386
|
if (filePath === ROUTING_FILE) {
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
console.log(`Created bucket ${key}`);
|
|
232
|
-
return;
|
|
233
|
-
}
|
|
234
|
-
// Routing writes bypass fast mode (the config must apply immediately and survive a crash)
|
|
235
|
-
// and ignore immutable (the routing file must always stay updatable). An older lastModified
|
|
236
|
-
// no-ops inside set, in which case the reload below sees no change.
|
|
237
|
-
await loaded.store.set(ROUTING_FILE, data, { lastModified: config?.lastModified });
|
|
238
|
-
await scheduleRoutingReload(account, bucketName);
|
|
239
|
-
return;
|
|
387
|
+
let key = `${account}/${bucketName}`;
|
|
388
|
+
let run = (routingWrites.get(key) || Promise.resolve()).then(() => writeRoutingConfig(account, bucketName, data, config));
|
|
389
|
+
// The chain must survive a failed write, so the stored link swallows the error (the caller
|
|
390
|
+
// still gets it from run)
|
|
391
|
+
routingWrites.set(key, run.then(() => { }, () => { }));
|
|
392
|
+
return await run;
|
|
240
393
|
}
|
|
394
|
+
let loaded = await getLoadedBucket(account, bucketName);
|
|
241
395
|
if (!loaded) {
|
|
242
396
|
throw new Error(`Bucket ${account}/${bucketName} does not exist. Write its routing config to ${JSON.stringify(ROUTING_FILE)} to create it.`);
|
|
243
397
|
}
|
|
244
|
-
|
|
245
|
-
|
|
398
|
+
// The write time is determined exactly once, HERE at the first node that receives the write;
|
|
399
|
+
// it propagates as lastModified, so every node evaluates the bucket options at the same instant
|
|
400
|
+
let writeTime = config?.lastModified || Date.now();
|
|
401
|
+
// Only freshly-stamped writes are checked: a write with an explicit lastModified is
|
|
402
|
+
// synchronization/backfill (old data legitimately lands on sources whose window has moved on),
|
|
403
|
+
// but a fresh write reaching us outside our windows/routes means the client resolved its
|
|
404
|
+
// target from a stale view - it must re-resolve and retry at the correct source.
|
|
405
|
+
let route = getRoute(filePath);
|
|
406
|
+
if (!config?.lastModified && loaded.selfEntries.length) {
|
|
407
|
+
let timeValid = loaded.selfEntries.filter(x => writeTime >= x.validWindow[0] - WRITE_PAST_WINDOW_GRACE && writeTime <= x.validWindow[1] + WRITE_PAST_WINDOW_GRACE);
|
|
408
|
+
if (!timeValid.length) {
|
|
409
|
+
throw new Error(`${STORAGE_WRONG_VALID_WINDOW} This server is not a valid write target at ${writeTime} for bucket ${account}/${bucketName} (our valid windows: ${JSON.stringify(loaded.selfEntries.map(x => x.validWindow))}). Re-resolve the currently valid source and retry.`);
|
|
410
|
+
}
|
|
411
|
+
if (!timeValid.some(x => routeContains(x.route, route))) {
|
|
412
|
+
throw new Error(`${STORAGE_WRONG_ROUTE} This server does not handle route ${route} (key ${JSON.stringify(filePath)}) for bucket ${account}/${bucketName} (our routes at this time: ${JSON.stringify(timeValid.map(x => x.route || FULL_ROUTE))}). Re-resolve the source for this key and retry.`);
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
await assertMutable(loaded, filePath, writeTime);
|
|
416
|
+
await loaded.store.set(filePath, data, { ...getWriteConfig(loaded, writeTime, route), lastModified: writeTime });
|
|
246
417
|
}
|
|
247
418
|
|
|
248
419
|
export async function deleteBucketFile(account: string, bucketName: string, filePath: string): Promise<void> {
|
|
@@ -251,7 +422,7 @@ export async function deleteBucketFile(account: string, bucketName: string, file
|
|
|
251
422
|
}
|
|
252
423
|
let loaded = await getLoadedBucket(account, bucketName);
|
|
253
424
|
if (!loaded) return;
|
|
254
|
-
await loaded.store.del(filePath, getWriteConfig(loaded));
|
|
425
|
+
await loaded.store.del(filePath, getWriteConfig(loaded, Date.now(), getRoute(filePath)));
|
|
255
426
|
}
|
|
256
427
|
|
|
257
428
|
// ── local IArchives access ──
|
|
@@ -274,7 +445,7 @@ class ArchivesLocalBucket implements IArchives {
|
|
|
274
445
|
let result = await this.get2(fileName, config);
|
|
275
446
|
return result && result.data || undefined;
|
|
276
447
|
}
|
|
277
|
-
public async get2(fileName: string, config?: { range?: { start: number; end: number } }): Promise<{ data: Buffer; writeTime: number } | undefined> {
|
|
448
|
+
public async get2(fileName: string, config?: { range?: { start: number; end: number } }): Promise<{ data: Buffer; writeTime: number; size: number } | undefined> {
|
|
278
449
|
let bucket = await this.getBucket();
|
|
279
450
|
if (!bucket) return undefined;
|
|
280
451
|
return await bucket.store.get2(fileName, config);
|
|
@@ -291,7 +462,7 @@ class ArchivesLocalBucket implements IArchives {
|
|
|
291
462
|
if (!bucket) {
|
|
292
463
|
throw new Error(`Bucket ${this.account}/${this.bucketName} does not exist. Write its routing config to ${JSON.stringify(ROUTING_FILE)} to create it.`);
|
|
293
464
|
}
|
|
294
|
-
await assertMutable(bucket, config.path);
|
|
465
|
+
await assertMutable(bucket, config.path, Date.now());
|
|
295
466
|
let id = await bucket.store.startLargeUpload();
|
|
296
467
|
try {
|
|
297
468
|
while (true) {
|
|
@@ -327,7 +498,18 @@ class ArchivesLocalBucket implements IArchives {
|
|
|
327
498
|
public async getConfig(): Promise<ArchivesConfig> {
|
|
328
499
|
let bucket = await this.getBucket();
|
|
329
500
|
// Missing buckets say true, matching what they become once created (the default store type)
|
|
330
|
-
|
|
501
|
+
let progress = bucket?.store.getSyncProgress?.();
|
|
502
|
+
return {
|
|
503
|
+
supportsChangesAfter: !bucket || !!bucket.store.getChangesAfter,
|
|
504
|
+
remoteConfig: bucket?.routing,
|
|
505
|
+
index: progress?.index,
|
|
506
|
+
indexSources: progress?.sources,
|
|
507
|
+
readerDiskLimit: progress?.readerDiskLimit,
|
|
508
|
+
syncing: progress?.syncing,
|
|
509
|
+
};
|
|
510
|
+
}
|
|
511
|
+
public async hasWriteAccess(): Promise<boolean> {
|
|
512
|
+
return true;
|
|
331
513
|
}
|
|
332
514
|
public async getChangesAfter(time: number): Promise<ArchiveFileInfo[]> {
|
|
333
515
|
let bucket = await this.getBucket();
|
package/testsite/server.ts
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
process.env.NODE_ENV = "production";
|
|
2
|
-
import os from "os";
|
|
3
2
|
import path from "path";
|
|
4
3
|
import { SocketFunction } from "socket-function/SocketFunction";
|
|
5
4
|
import { RequireController } from "socket-function/require/RequireController";
|
|
@@ -30,7 +29,6 @@ async function main() {
|
|
|
30
29
|
await hostServer({
|
|
31
30
|
domain: DOMAIN,
|
|
32
31
|
port: PORT,
|
|
33
|
-
cloudflareApiToken: { path: os.homedir() + "/vidgridweb.com.key" },
|
|
34
32
|
setDNSRecord: true,
|
|
35
33
|
});
|
|
36
34
|
}
|
package/yarn.lock
CHANGED
|
@@ -1944,10 +1944,10 @@ slash@^3.0.0:
|
|
|
1944
1944
|
resolved "https://registry.yarnpkg.com/slash/-/slash-3.0.0.tgz#6539be870c165adbd5240220dbe361f1bc4d4634"
|
|
1945
1945
|
integrity sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==
|
|
1946
1946
|
|
|
1947
|
-
socket-function@^1.2.
|
|
1948
|
-
version "1.2.
|
|
1949
|
-
resolved "https://registry.yarnpkg.com/socket-function/-/socket-function-1.2.
|
|
1950
|
-
integrity sha512-
|
|
1947
|
+
socket-function@^1.2.16:
|
|
1948
|
+
version "1.2.16"
|
|
1949
|
+
resolved "https://registry.yarnpkg.com/socket-function/-/socket-function-1.2.16.tgz#6ad7fc1971202d9768582b0212f086675fba61b5"
|
|
1950
|
+
integrity sha512-o3GATr5Fa+496StsPWJIb0yytY/XRMZjJQ4cgb1LOQIfFv7uJUJ3NZtcldMeN/kOmqI0hBaMvhRLC3VHke9awA==
|
|
1951
1951
|
dependencies:
|
|
1952
1952
|
"@types/pako" "^2.0.3"
|
|
1953
1953
|
"@types/ws" "^8.5.3"
|
|
@@ -1957,7 +1957,7 @@ socket-function@^1.2.4:
|
|
|
1957
1957
|
preact "10.24.3"
|
|
1958
1958
|
typedev "^0.4.0"
|
|
1959
1959
|
typenode "^6.0.1"
|
|
1960
|
-
ws "^8.
|
|
1960
|
+
ws "^8.21.0"
|
|
1961
1961
|
optionalDependencies:
|
|
1962
1962
|
lmdb "^3.5.1"
|
|
1963
1963
|
|
|
@@ -2177,11 +2177,16 @@ wrappy@1:
|
|
|
2177
2177
|
resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"
|
|
2178
2178
|
integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==
|
|
2179
2179
|
|
|
2180
|
-
ws@^8.
|
|
2180
|
+
ws@^8.18.3:
|
|
2181
2181
|
version "8.18.3"
|
|
2182
2182
|
resolved "https://registry.yarnpkg.com/ws/-/ws-8.18.3.tgz#b56b88abffde62791c639170400c93dcb0c95472"
|
|
2183
2183
|
integrity sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==
|
|
2184
2184
|
|
|
2185
|
+
ws@^8.21.0:
|
|
2186
|
+
version "8.21.1"
|
|
2187
|
+
resolved "https://registry.yarnpkg.com/ws/-/ws-8.21.1.tgz#045650cd4b1207809e7547146223c3814a9af586"
|
|
2188
|
+
integrity sha512-+0NTnW77fFN/DjQi6k/Sq/Yvk4Sgajw7urW8V+asjXnRgDs9gyGkdb7EzgfhA4goXsRIZKE28fzIXBHEzhuiWw==
|
|
2189
|
+
|
|
2185
2190
|
y18n@^4.0.0:
|
|
2186
2191
|
version "4.0.3"
|
|
2187
2192
|
resolved "https://registry.yarnpkg.com/y18n/-/y18n-4.0.3.tgz#b5f259c82cd6e336921efd7bfd8bf560de9eeedf"
|