sliftutils 1.7.13 → 1.7.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/settings.local.json +2 -1
- package/index.d.ts +64 -38
- package/misc/dist/getSecret.ts.cache +2 -2
- package/misc/dist/strings.ts.cache +2 -2
- package/misc/https/dist/certs.ts.cache +2 -2
- package/misc/https/dist/persistentLocalStorage.ts.cache +2 -2
- package/misc/https/hostServer.d.ts +7 -0
- package/misc/https/hostServer.ts +89 -20
- package/misc/openrouter.ts +17 -14
- package/package.json +2 -2
- package/storage/ArchivesDisk.d.ts +1 -1
- package/storage/ArchivesDisk.ts +2 -1
- package/storage/BulkDatabase2/dist/BulkDatabaseBase.ts.cache +2 -2
- package/storage/BulkDatabase2/dist/BulkDatabaseFormat.ts.cache +2 -2
- package/storage/BulkDatabase2/dist/BulkDatabaseMerge.ts.cache +2 -2
- package/storage/BulkDatabase2/dist/BulkDatabaseReader.ts.cache +2 -2
- package/storage/BulkDatabase2/dist/LoadedIndex.ts.cache +2 -2
- package/storage/BulkDatabase2/dist/WriteOverlay.ts.cache +2 -2
- package/storage/BulkDatabase2/dist/blockCache.ts.cache +2 -2
- package/storage/BulkDatabase2/dist/mergeLock.ts.cache +2 -2
- package/storage/BulkDatabase2/dist/mergeMarkers.ts.cache +2 -2
- package/storage/BulkDatabase2/dist/streamLog.ts.cache +2 -2
- package/storage/BulkDatabase2/dist/syncClient.ts.cache +2 -2
- package/storage/IArchives.d.ts +5 -1
- package/storage/IArchives.ts +5 -1
- package/storage/backblaze.d.ts +1 -1
- package/storage/backblaze.ts +9 -6
- package/storage/remoteStorage/ArchivesRemote.d.ts +1 -1
- package/storage/remoteStorage/ArchivesRemote.ts +2 -1
- package/storage/remoteStorage/ArchivesUrl.d.ts +1 -1
- package/storage/remoteStorage/ArchivesUrl.ts +13 -9
- package/storage/remoteStorage/blobStore.d.ts +6 -2
- package/storage/remoteStorage/blobStore.ts +46 -6
- package/storage/remoteStorage/createArchives.d.ts +10 -11
- package/storage/remoteStorage/createArchives.ts +74 -26
- package/storage/remoteStorage/deployTakeover.d.ts +24 -0
- package/storage/remoteStorage/deployTakeover.ts +372 -0
- package/storage/remoteStorage/dist/ArchivesUrl.ts.cache +16 -11
- package/storage/remoteStorage/dist/blobStore.ts.cache +39 -4
- package/storage/remoteStorage/dist/createArchives.ts.cache +91 -9
- package/storage/remoteStorage/dist/deployTakeover.ts.cache +371 -0
- package/storage/remoteStorage/dist/remoteConfig.ts.cache +9 -3
- package/storage/remoteStorage/dist/sourceWrapper.ts.cache +3 -3
- package/storage/remoteStorage/dist/storageController.ts.cache +11 -12
- package/storage/remoteStorage/dist/storageServerState.ts.cache +120 -19
- package/storage/remoteStorage/remoteConfig.d.ts +1 -0
- package/storage/remoteStorage/remoteConfig.ts +6 -0
- package/storage/remoteStorage/storageServer.ts +15 -1
- package/storage/remoteStorage/storageServerState.d.ts +3 -0
- package/storage/remoteStorage/storageServerState.ts +55 -8
- package/yarn.lock +4 -4
- package/storage/dist/faceFramesData.ts.cache +0 -55
- package/storage/embeddingBench.d.ts +0 -1
- package/storage/embeddingBench.ts +0 -152
- package/storage/faceFramesData.d.ts +0 -1
- package/storage/faceFramesData.ts +0 -49
- package/storage/faceFramesServer.d.ts +0 -1
- package/storage/faceFramesServer.ts +0 -56
- package/storage/proxydatabase/ivfDbCheck.d.ts +0 -1
- package/storage/proxydatabase/ivfDbCheck.ts +0 -85
|
@@ -17,9 +17,14 @@ export declare class ArchivesChain implements IArchives {
|
|
|
17
17
|
private init;
|
|
18
18
|
private buildSources;
|
|
19
19
|
private startConfigPoll;
|
|
20
|
+
private configRefreshInFlight;
|
|
21
|
+
private refreshActiveConfig;
|
|
22
|
+
private fetchLatestConfig;
|
|
20
23
|
private checkForNewConfig;
|
|
21
24
|
private run;
|
|
22
25
|
private runWrite;
|
|
26
|
+
private lastConfigRefresh;
|
|
27
|
+
private prepareWrongTargetRetry;
|
|
23
28
|
private request;
|
|
24
29
|
waitingForAccess(): Promise<{
|
|
25
30
|
link: string;
|
|
@@ -62,20 +67,14 @@ export declare class ArchivesChain implements IArchives {
|
|
|
62
67
|
/** True only when EVERY write-receiving source would accept our writes (partial write access
|
|
63
68
|
* desynchronizes sources, so it counts as no access). */
|
|
64
69
|
hasWriteAccess(): Promise<boolean>;
|
|
65
|
-
|
|
70
|
+
/** Returns the full key written. Plain keys come back unchanged; keys containing VARIABLE_SHARD
|
|
71
|
+
* are automatically materialized (a shard value is picked and embedded, see setVariableShard)
|
|
72
|
+
* and the caller needs the returned key to ever read the value back. */
|
|
66
73
|
set(fileName: string, data: Buffer, config?: {
|
|
67
74
|
lastModified?: number;
|
|
68
|
-
}): Promise<void>;
|
|
69
|
-
del(fileName: string): Promise<void>;
|
|
70
|
-
/** Writes a key containing the VARIABLE_SHARD sentinel: picks the lowest-latency up write
|
|
71
|
-
* shard, materializes the key with a random value inside that shard's route, writes it, and
|
|
72
|
-
* returns the FULL key actually written (the caller needs it to ever read the value back).
|
|
73
|
-
* Unlike normal writes this CAN move to another shard when the preferred one is down (error +
|
|
74
|
-
* socket down, same rule as reads) - each shard receives a different key, so write
|
|
75
|
-
* consistency is preserved. */
|
|
76
|
-
setVariableShard(key: string, data: Buffer, config?: {
|
|
77
|
-
lastModified?: number;
|
|
78
75
|
}): Promise<string>;
|
|
76
|
+
del(fileName: string): Promise<void>;
|
|
77
|
+
private setVariableShard;
|
|
79
78
|
setLargeFile(config: {
|
|
80
79
|
path: string;
|
|
81
80
|
getNextData(): Promise<Buffer | undefined>;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
module.allowclient = true;
|
|
2
2
|
|
|
3
3
|
import { isNode, sort } from "socket-function/src/misc";
|
|
4
|
+
import { delay } from "socket-function/src/batching";
|
|
4
5
|
import {
|
|
5
6
|
IArchives, RemoteConfig, RemoteConfigBase, HostedConfig, BackblazeConfig,
|
|
6
7
|
ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, WRITE_PAST_WINDOW_GRACE, STORAGE_WRONG_VALID_WINDOW,
|
|
@@ -28,6 +29,13 @@ import { SourceWrapper, RETRY_START_DELAY, RETRY_MAX_DELAY, RETRY_GROWTH } from
|
|
|
28
29
|
// rebuild the source list when it changed (reusing wrappers for unchanged sources).
|
|
29
30
|
|
|
30
31
|
const CONFIG_POLL_INTERVAL = 5 * 60 * 1000;
|
|
32
|
+
// Wrong-valid-window rejections within this distance of a known window boundary are just us racing
|
|
33
|
+
// the boundary itself - waiting fixes them, no new config needed
|
|
34
|
+
const WRONG_TARGET_BOUNDARY_WINDOW = 30 * 1000;
|
|
35
|
+
const WRONG_TARGET_BOUNDARY_RETRY_DELAY = 15 * 1000;
|
|
36
|
+
// Otherwise our config is stale; re-fetch it from the server, at most this often (when throttled,
|
|
37
|
+
// retry with what we have)
|
|
38
|
+
const CONFIG_REFRESH_THROTTLE = 30 * 1000;
|
|
31
39
|
|
|
32
40
|
// The direct API IArchives for one source, with no URL-form fallback and no chaining. Used by the
|
|
33
41
|
// storage server for its synchronization sources. Denied calls throw immediately (registering the
|
|
@@ -241,13 +249,39 @@ export class ArchivesChain implements IArchives {
|
|
|
241
249
|
private startConfigPoll(): void {
|
|
242
250
|
if (this.pollTimer || this.disposed) return;
|
|
243
251
|
this.pollTimer = setInterval(() => {
|
|
244
|
-
void this.
|
|
252
|
+
void this.refreshActiveConfig().catch((e: Error) => {
|
|
245
253
|
console.error(`Checking for a new storage routing config failed for ${this.getDebugName()}: ${e.stack ?? e}`);
|
|
246
254
|
});
|
|
247
255
|
}, CONFIG_POLL_INTERVAL);
|
|
248
256
|
(this.pollTimer as { unref?: () => void }).unref?.();
|
|
249
257
|
}
|
|
250
258
|
|
|
259
|
+
// Deduplicates concurrent refreshes (the poll timer and wrong-target write retries share this)
|
|
260
|
+
private configRefreshInFlight: Promise<void> | undefined;
|
|
261
|
+
private refreshActiveConfig(): Promise<void> {
|
|
262
|
+
if (!this.configRefreshInFlight) {
|
|
263
|
+
this.configRefreshInFlight = this.checkForNewConfig().finally(() => {
|
|
264
|
+
this.configRefreshInFlight = undefined;
|
|
265
|
+
});
|
|
266
|
+
}
|
|
267
|
+
return this.configRefreshInFlight;
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
// The latest config, as the server INTERPRETS it: getConfig carries in-memory overlays (deploy
|
|
271
|
+
// takeover remaps) that are deliberately never written into the routing file, so it's
|
|
272
|
+
// preferred over reading the raw file. URL-only chains fall back to the raw file.
|
|
273
|
+
private async fetchLatestConfig(state: ChainState): Promise<RemoteConfig | undefined> {
|
|
274
|
+
try {
|
|
275
|
+
let config = await this.run(state, { apiOnly: true }, archives => archives.getConfig());
|
|
276
|
+
if (config.remoteConfig) {
|
|
277
|
+
return normalizeRemoteConfig(config.remoteConfig);
|
|
278
|
+
}
|
|
279
|
+
} catch { }
|
|
280
|
+
let data = await this.run(state, {}, archives => archives.get(ROUTING_FILE));
|
|
281
|
+
if (!data) return undefined;
|
|
282
|
+
return parseRoutingData(data);
|
|
283
|
+
}
|
|
284
|
+
|
|
251
285
|
private async checkForNewConfig(): Promise<void> {
|
|
252
286
|
if (this.disposed || !this.statePromise) return;
|
|
253
287
|
let state: ChainState;
|
|
@@ -257,9 +291,8 @@ export class ArchivesChain implements IArchives {
|
|
|
257
291
|
// Init is failing; its own retry loop handles that
|
|
258
292
|
return;
|
|
259
293
|
}
|
|
260
|
-
let
|
|
261
|
-
if (!
|
|
262
|
-
let latest = parseRoutingData(data);
|
|
294
|
+
let latest = await this.fetchLatestConfig(state);
|
|
295
|
+
if (!latest) return;
|
|
263
296
|
if (JSON.stringify(latest) === JSON.stringify(state.config)) return;
|
|
264
297
|
console.log(`Storage routing config changed for ${this.getDebugName()}, rebuilding sources`);
|
|
265
298
|
let oldByConfig = new Map(state.sources.map(source => [JSON.stringify(source.config), source]));
|
|
@@ -289,7 +322,7 @@ export class ArchivesChain implements IArchives {
|
|
|
289
322
|
// an application error and throws as-is. A down source gets its background reconnect kicked.
|
|
290
323
|
private async run<T>(state: ChainState, config: { apiOnly?: boolean; write?: boolean; route?: number }, run: (archives: IArchives) => Promise<T>): Promise<T> {
|
|
291
324
|
if (config.write) {
|
|
292
|
-
return await this.runWrite(
|
|
325
|
+
return await this.runWrite(config.route, run);
|
|
293
326
|
}
|
|
294
327
|
let errors: string[] = [];
|
|
295
328
|
for (let source of state.sources) {
|
|
@@ -324,10 +357,11 @@ export class ArchivesChain implements IArchives {
|
|
|
324
357
|
// scatter its writes across the chain. (Reads fail over freely above: sources are synchronized
|
|
325
358
|
// copies, so reading from any of them is safe.) The one retry is for a window boundary passing
|
|
326
359
|
// (or a route disagreement) mid-write, which re-resolves the target rather than falling back.
|
|
327
|
-
private async runWrite<T>(
|
|
360
|
+
private async runWrite<T>(route: number | undefined, run: (archives: IArchives) => Promise<T>): Promise<T> {
|
|
328
361
|
let retriedWrongWindow = false;
|
|
329
362
|
let retriedWrongRoute = false;
|
|
330
363
|
while (true) {
|
|
364
|
+
let state = await this.getState();
|
|
331
365
|
let target = state.sources.find(x => configAcceptsWrites(x.config) && (route === undefined || routeContains(x.config.route, route)));
|
|
332
366
|
if (!target) {
|
|
333
367
|
throw new Error(`No source accepts writes for ${this.getDebugName()}${route !== undefined && ` (route ${route})` || ""} (every source is outside its valid window or outside the key's route)`);
|
|
@@ -342,10 +376,12 @@ export class ArchivesChain implements IArchives {
|
|
|
342
376
|
let message = String((e as Error).stack ?? e);
|
|
343
377
|
if (message.includes(STORAGE_WRONG_VALID_WINDOW) && !retriedWrongWindow) {
|
|
344
378
|
retriedWrongWindow = true;
|
|
379
|
+
await this.prepareWrongTargetRetry(state, "window");
|
|
345
380
|
continue;
|
|
346
381
|
}
|
|
347
382
|
if (message.includes(STORAGE_WRONG_ROUTE) && !retriedWrongRoute) {
|
|
348
383
|
retriedWrongRoute = true;
|
|
384
|
+
await this.prepareWrongTargetRetry(state, "route");
|
|
349
385
|
continue;
|
|
350
386
|
}
|
|
351
387
|
if (!target.isConnected()) {
|
|
@@ -356,6 +392,23 @@ export class ArchivesChain implements IArchives {
|
|
|
356
392
|
}
|
|
357
393
|
}
|
|
358
394
|
|
|
395
|
+
private lastConfigRefresh = 0;
|
|
396
|
+
// A wrong-target rejection means either we raced a window boundary (time fixes it) or our
|
|
397
|
+
// config is stale - e.g. a deploy-takeover remap that only exists in the server's memory
|
|
398
|
+
private async prepareWrongTargetRetry(state: ChainState, kind: "window" | "route"): Promise<void> {
|
|
399
|
+
if (kind === "window") {
|
|
400
|
+
let now = Date.now();
|
|
401
|
+
let nearBoundary = state.sources.some(source => source.config.validWindow.some(t => t > 0 && t < Number.MAX_SAFE_INTEGER && Math.abs(t - now) <= WRONG_TARGET_BOUNDARY_WINDOW));
|
|
402
|
+
if (nearBoundary) {
|
|
403
|
+
await delay(WRONG_TARGET_BOUNDARY_RETRY_DELAY);
|
|
404
|
+
return;
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
if (Date.now() - this.lastConfigRefresh < CONFIG_REFRESH_THROTTLE) return;
|
|
408
|
+
this.lastConfigRefresh = Date.now();
|
|
409
|
+
await this.refreshActiveConfig();
|
|
410
|
+
}
|
|
411
|
+
|
|
359
412
|
private async request<T>(config: { apiOnly?: boolean; write?: boolean; route?: number }, run: (archives: IArchives) => Promise<T>): Promise<T> {
|
|
360
413
|
let state = await this.getState();
|
|
361
414
|
return await this.run(state, config, run);
|
|
@@ -496,33 +549,26 @@ export class ArchivesChain implements IArchives {
|
|
|
496
549
|
return true;
|
|
497
550
|
}
|
|
498
551
|
|
|
499
|
-
|
|
552
|
+
/** Returns the full key written. Plain keys come back unchanged; keys containing VARIABLE_SHARD
|
|
553
|
+
* are automatically materialized (a shard value is picked and embedded, see setVariableShard)
|
|
554
|
+
* and the caller needs the returned key to ever read the value back. */
|
|
555
|
+
public async set(fileName: string, data: Buffer, config?: { lastModified?: number }): Promise<string> {
|
|
500
556
|
if (fileName.includes(VARIABLE_SHARD) && parseVariableRoute(fileName) === undefined) {
|
|
501
|
-
|
|
557
|
+
return await this.setVariableShard(fileName, data, config);
|
|
502
558
|
}
|
|
503
|
-
}
|
|
504
|
-
|
|
505
|
-
public async set(fileName: string, data: Buffer, config?: { lastModified?: number }): Promise<void> {
|
|
506
|
-
this.assertNotBareVariableShard(fileName);
|
|
507
559
|
await this.request({ write: true, route: getRoute(fileName) }, archives => archives.set(fileName, data, config));
|
|
560
|
+
return fileName;
|
|
508
561
|
}
|
|
509
562
|
public async del(fileName: string): Promise<void> {
|
|
510
563
|
await this.request({ write: true, route: getRoute(fileName) }, archives => archives.del(fileName));
|
|
511
564
|
}
|
|
512
565
|
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
public async setVariableShard(key: string, data: Buffer, config?: { lastModified?: number }): Promise<string> {
|
|
520
|
-
if (!key.includes(VARIABLE_SHARD)) {
|
|
521
|
-
throw new Error(`Expected the key to contain the VARIABLE_SHARD sentinel, was ${JSON.stringify(key)}`);
|
|
522
|
-
}
|
|
523
|
-
if (parseVariableRoute(key) !== undefined) {
|
|
524
|
-
throw new Error(`The key already has a materialized shard value; write it with set instead. Key: ${JSON.stringify(key)}`);
|
|
525
|
-
}
|
|
566
|
+
// Writes a bare variable-shard key: picks the lowest-latency up write shard, materializes the
|
|
567
|
+
// key with a random value inside that shard's route, writes it, and returns the FULL key
|
|
568
|
+
// actually written. Unlike normal writes this CAN move to another shard when the preferred one
|
|
569
|
+
// is down (error + socket down, same rule as reads) - each shard receives a different key, so
|
|
570
|
+
// write consistency is preserved.
|
|
571
|
+
private async setVariableShard(key: string, data: Buffer, config?: { lastModified?: number }): Promise<string> {
|
|
526
572
|
let state = await this.getState();
|
|
527
573
|
// Per-shard write consistency still holds: within a route, only the FIRST source may take
|
|
528
574
|
// writes - latency only picks WHICH shard the key materializes into
|
|
@@ -560,7 +606,9 @@ export class ArchivesChain implements IArchives {
|
|
|
560
606
|
}
|
|
561
607
|
|
|
562
608
|
public async setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined> }): Promise<void> {
|
|
563
|
-
|
|
609
|
+
if (config.path.includes(VARIABLE_SHARD) && parseVariableRoute(config.path) === undefined) {
|
|
610
|
+
throw new Error(`setLargeFile does not support VARIABLE_SHARD keys (there is no way to return the materialized key); write the file with set, or materialize the key yourself. Key: ${JSON.stringify(config.path)}`);
|
|
611
|
+
}
|
|
564
612
|
let state = await this.getState();
|
|
565
613
|
// Same consistency rule as runWrite: the first currently-valid source for the route, or nothing
|
|
566
614
|
let route = getRoute(config.path);
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { RemoteConfig } from "../IArchives";
|
|
2
|
+
export type TakeoverEvent = "remapChanged" | "diskScan";
|
|
3
|
+
/** Starts the takeover machinery. Port fallback (alternate port + registry + acquisition polling)
|
|
4
|
+
* works regardless; without a deploy timeline folder the switchover-specific parts (the remap,
|
|
5
|
+
* the flush deadline, the tighter acquisition pacing) simply stay inert. */
|
|
6
|
+
export declare function initDeployTakeover(config: {
|
|
7
|
+
domain: string;
|
|
8
|
+
mainPort: number;
|
|
9
|
+
storageFolder: string;
|
|
10
|
+
}): Promise<void>;
|
|
11
|
+
/** Called when we had to listen on an alternate port (the main port was still held by our
|
|
12
|
+
* predecessor): registers it so the predecessor can route the middle overlap window to us. */
|
|
13
|
+
export declare function registerAltPort(port: number): Promise<void>;
|
|
14
|
+
export declare function onTakeoverEvent(listener: (event: TakeoverEvent) => void): void;
|
|
15
|
+
/** The interpretation overlay: splits every source pointing at our domain+main port so the middle
|
|
16
|
+
* of the deploy overlap points at the alternate port. Pure, in-memory only - the stored routing
|
|
17
|
+
* config is never modified, and this must never be applied to data that gets persisted. */
|
|
18
|
+
export declare function applyDeployRemap(routing: RemoteConfig): RemoteConfig;
|
|
19
|
+
/** For the dying process: fast-write flush delays must never extend past this time, and after it
|
|
20
|
+
* fast writes flush immediately - so nothing is left in memory when the write window transfers. */
|
|
21
|
+
export declare function getFlushDeadline(): number | undefined;
|
|
22
|
+
/** How long to wait between main-port acquisition attempts: tight around the predecessor's
|
|
23
|
+
* scheduled death (when the port actually frees), relaxed otherwise. */
|
|
24
|
+
export declare function getMainPortAcquireDelay(): number;
|
|
@@ -0,0 +1,372 @@
|
|
|
1
|
+
import fs from "fs";
|
|
2
|
+
import path from "path";
|
|
3
|
+
import { exec } from "child_process";
|
|
4
|
+
import { RemoteConfig, RemoteConfigBase, HostedConfig } from "../IArchives";
|
|
5
|
+
import { parseHostedUrl, replaceHostedUrlPort } from "./remoteConfig";
|
|
6
|
+
|
|
7
|
+
// Deploy takeover: the deploy manager (querysub's parametersTimeline) writes <n>-parameters.json
|
|
8
|
+
// files into our instance folder describing when each version runs and when it is shut down. Old
|
|
9
|
+
// and new version overlap for the overlap time (O = oldEnd - newStart). We turn that overlap into
|
|
10
|
+
// an atomic write handoff by REINTERPRETING every routing source that points at us: its valid
|
|
11
|
+
// window is split so that the middle of the overlap ([S + 0.5*O, S + 1.5*O]) points at the new
|
|
12
|
+
// process's temporary alternate port. The existing valid-window machinery then does everything -
|
|
13
|
+
// the old process starts rejecting fresh writes at the first boundary, clients re-resolve, and the
|
|
14
|
+
// new process (which recognizes the alternate port as itself) takes over. The remap is an
|
|
15
|
+
// INTERPRETATION OVERLAY, applied where routing configs are used (see applyDeployRemap callers) -
|
|
16
|
+
// it is never written to disk, never synchronized, and never part of any stored configuration.
|
|
17
|
+
//
|
|
18
|
+
// Everything here is recomputed idempotently from its inputs (timeline files, the port registry,
|
|
19
|
+
// our ancestor pids, the clock) - files can appear or change at any moment, and we can start at
|
|
20
|
+
// any point during a switchover.
|
|
21
|
+
|
|
22
|
+
const PARAMETERS_TIMELINE_FILE_REGEX = /^(\d+)-parameters\.json$/;
|
|
23
|
+
const ALIVE_WINDOW_FOREVER = Number.MAX_SAFE_INTEGER;
|
|
24
|
+
const TIMELINE_POLL_INTERVAL = 60 * 1000;
|
|
25
|
+
// Processes listening on an alternate port register it here (sibling of the storage folder), keyed
|
|
26
|
+
// by their actual pid; content includes their ancestor pids so the timeline entry pid (a shell
|
|
27
|
+
// ancestor of the process) can be matched to a registry entry
|
|
28
|
+
const PORT_REGISTRY_FOLDER_NAME = "storagePortRegistry";
|
|
29
|
+
// Main-port acquisition attempts run this often, tightening near the predecessor's scheduled death
|
|
30
|
+
const ACQUIRE_SLOW_DELAY = 30 * 1000;
|
|
31
|
+
const ACQUIRE_FAST_DELAY = 5 * 1000;
|
|
32
|
+
const ACQUIRE_FAST_WINDOW = 60 * 1000;
|
|
33
|
+
// Disk rescans fire this far before and after the write handoff, catching files the other process
|
|
34
|
+
// wrote (the BulkDatabase2 index is atomic but its entries land at write time, not file time)
|
|
35
|
+
const SWITCH_SCAN_LEAD = 10 * 1000;
|
|
36
|
+
// The dying process must have flushed everything before the handoff: fast-write delays never
|
|
37
|
+
// extend past S + overlap * this fraction, and after that point fast writes flush immediately
|
|
38
|
+
const FLUSH_DEADLINE_FRACTION = 0.25;
|
|
39
|
+
const BOUNDARY_A_FRACTION = 0.5;
|
|
40
|
+
const BOUNDARY_B_FRACTION = 1.5;
|
|
41
|
+
const REMAP_EXPIRE_FRACTION = 2;
|
|
42
|
+
|
|
43
|
+
type TimelineEntry = {
|
|
44
|
+
pid?: number;
|
|
45
|
+
aliveWindow: [number, number];
|
|
46
|
+
parameters: unknown;
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
type TakeoverComputed = {
|
|
50
|
+
// We are scheduled to die with a successor overlapping us (known from the files alone, before
|
|
51
|
+
// the successor's port is discovered - the flush deadline applies immediately)
|
|
52
|
+
dying?: { successorStart: number; overlap: number };
|
|
53
|
+
// The window reinterpretation, once the middle window's port is known (the successor's
|
|
54
|
+
// registered alternate port - or our own, when we ARE the successor)
|
|
55
|
+
remap?: { boundaryA: number; boundaryB: number; expire: number; altPort: number };
|
|
56
|
+
// When our predecessor dies and frees the main port (for acquisition pacing)
|
|
57
|
+
predecessorEnd?: number;
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
export type TakeoverEvent = "remapChanged" | "diskScan";
|
|
61
|
+
|
|
62
|
+
let initialized: { domain: string; mainPort: number; storageFolder: string } | undefined;
|
|
63
|
+
let ancestorPids = new Set<number>();
|
|
64
|
+
let timelineFolder: string | undefined;
|
|
65
|
+
let ourAltPort: number | undefined;
|
|
66
|
+
let current: TakeoverComputed = {};
|
|
67
|
+
let currentKey = JSON.stringify(current);
|
|
68
|
+
let eventListeners: ((event: TakeoverEvent) => void)[] = [];
|
|
69
|
+
let eventTimers: ReturnType<typeof setTimeout>[] = [];
|
|
70
|
+
|
|
71
|
+
function execText(command: string): Promise<string> {
|
|
72
|
+
return new Promise((resolve, reject) => {
|
|
73
|
+
exec(command, (error, stdout) => {
|
|
74
|
+
if (error) {
|
|
75
|
+
reject(error);
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
resolve(stdout);
|
|
79
|
+
});
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// The timeline entry pid is the screen shell that spawned us - an ancestor - so we walk our whole
|
|
84
|
+
// ancestor chain to find which entry is us
|
|
85
|
+
async function getAncestorPids(): Promise<Set<number>> {
|
|
86
|
+
let pids = new Set<number>([process.pid]);
|
|
87
|
+
let currentPid = process.pid;
|
|
88
|
+
while (true) {
|
|
89
|
+
let ppid: number;
|
|
90
|
+
if (currentPid === process.pid && process.ppid) {
|
|
91
|
+
ppid = process.ppid;
|
|
92
|
+
} else {
|
|
93
|
+
try {
|
|
94
|
+
ppid = parseInt((await execText(`ps -o ppid= -p ${currentPid}`)).trim());
|
|
95
|
+
} catch {
|
|
96
|
+
break;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
if (!ppid || Number.isNaN(ppid) || ppid <= 1 || pids.has(ppid)) break;
|
|
100
|
+
pids.add(ppid);
|
|
101
|
+
currentPid = ppid;
|
|
102
|
+
}
|
|
103
|
+
return pids;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
async function findTimelineFolder(): Promise<string | undefined> {
|
|
107
|
+
let dir = process.cwd();
|
|
108
|
+
while (true) {
|
|
109
|
+
let files: string[] = [];
|
|
110
|
+
try {
|
|
111
|
+
files = await fs.promises.readdir(dir);
|
|
112
|
+
} catch { }
|
|
113
|
+
if (files.some(file => PARAMETERS_TIMELINE_FILE_REGEX.test(file))) {
|
|
114
|
+
return dir.replaceAll("\\", "/") + "/";
|
|
115
|
+
}
|
|
116
|
+
let parent = path.dirname(dir);
|
|
117
|
+
if (parent === dir) return undefined;
|
|
118
|
+
dir = parent;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
function pidAlive(pid: number): boolean {
|
|
123
|
+
try {
|
|
124
|
+
process.kill(pid, 0);
|
|
125
|
+
return true;
|
|
126
|
+
} catch (e) {
|
|
127
|
+
return (e as { code?: string }).code === "EPERM";
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function getRegistryFolder(): string {
|
|
132
|
+
if (!initialized) {
|
|
133
|
+
throw new Error(`Deploy takeover is not initialized`);
|
|
134
|
+
}
|
|
135
|
+
return path.join(path.dirname(path.resolve(initialized.storageFolder)), PORT_REGISTRY_FOLDER_NAME);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
type RegistryEntry = { pid: number; port: number; ancestorPids: number[] };
|
|
139
|
+
|
|
140
|
+
async function readPortRegistry(): Promise<RegistryEntry[]> {
|
|
141
|
+
let folder = getRegistryFolder();
|
|
142
|
+
let files: string[] = [];
|
|
143
|
+
try {
|
|
144
|
+
files = await fs.promises.readdir(folder);
|
|
145
|
+
} catch {
|
|
146
|
+
return [];
|
|
147
|
+
}
|
|
148
|
+
let entries: RegistryEntry[] = [];
|
|
149
|
+
for (let file of files) {
|
|
150
|
+
let match = /^(\d+)\.json$/.exec(file);
|
|
151
|
+
if (!match) continue;
|
|
152
|
+
let pid = parseInt(match[1]);
|
|
153
|
+
if (!pidAlive(pid)) {
|
|
154
|
+
try {
|
|
155
|
+
await fs.promises.unlink(path.join(folder, file));
|
|
156
|
+
} catch { }
|
|
157
|
+
continue;
|
|
158
|
+
}
|
|
159
|
+
try {
|
|
160
|
+
let entry = JSON.parse(await fs.promises.readFile(path.join(folder, file), "utf8")) as RegistryEntry;
|
|
161
|
+
entries.push({ ...entry, pid });
|
|
162
|
+
} catch { }
|
|
163
|
+
}
|
|
164
|
+
return entries;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
async function readTimelineEntries(): Promise<TimelineEntry[]> {
|
|
168
|
+
if (!timelineFolder) return [];
|
|
169
|
+
let entries: TimelineEntry[] = [];
|
|
170
|
+
for (let file of await fs.promises.readdir(timelineFolder)) {
|
|
171
|
+
if (!PARAMETERS_TIMELINE_FILE_REGEX.test(file)) continue;
|
|
172
|
+
try {
|
|
173
|
+
entries.push(JSON.parse(await fs.promises.readFile(timelineFolder + file, "utf8")) as TimelineEntry);
|
|
174
|
+
} catch (e) {
|
|
175
|
+
console.warn(`Could not read deploy timeline file ${timelineFolder + file}: ${(e as Error).stack ?? e}`);
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
return entries;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
function makeRemap(successorStart: number, overlap: number, altPort: number): TakeoverComputed["remap"] {
|
|
182
|
+
return {
|
|
183
|
+
boundaryA: successorStart + overlap * BOUNDARY_A_FRACTION,
|
|
184
|
+
boundaryB: successorStart + overlap * BOUNDARY_B_FRACTION,
|
|
185
|
+
expire: successorStart + overlap * REMAP_EXPIRE_FRACTION,
|
|
186
|
+
altPort,
|
|
187
|
+
};
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
function emit(event: TakeoverEvent): void {
|
|
191
|
+
for (let listener of eventListeners) {
|
|
192
|
+
try {
|
|
193
|
+
listener(event);
|
|
194
|
+
} catch (e) {
|
|
195
|
+
console.error(`Deploy takeover listener failed: ${(e as Error).stack ?? e}`);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
function scheduleEvents(): void {
|
|
201
|
+
for (let timer of eventTimers) {
|
|
202
|
+
clearTimeout(timer);
|
|
203
|
+
}
|
|
204
|
+
eventTimers = [];
|
|
205
|
+
let now = Date.now();
|
|
206
|
+
let schedule = (time: number, fnc: () => void) => {
|
|
207
|
+
if (time <= now) return;
|
|
208
|
+
let timer = setTimeout(fnc, time - now);
|
|
209
|
+
(timer as { unref?: () => void }).unref?.();
|
|
210
|
+
eventTimers.push(timer);
|
|
211
|
+
};
|
|
212
|
+
let remap = current.remap;
|
|
213
|
+
if (remap) {
|
|
214
|
+
schedule(remap.boundaryA - SWITCH_SCAN_LEAD, () => emit("diskScan"));
|
|
215
|
+
schedule(remap.boundaryA + SWITCH_SCAN_LEAD, () => emit("diskScan"));
|
|
216
|
+
schedule(remap.expire, () => void recompute().catch((e: Error) => console.error(`Deploy takeover recompute failed: ${e.stack ?? e}`)));
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
async function recompute(): Promise<void> {
|
|
221
|
+
if (!initialized || !timelineFolder) return;
|
|
222
|
+
let now = Date.now();
|
|
223
|
+
let entries = await readTimelineEntries();
|
|
224
|
+
let registry = await readPortRegistry();
|
|
225
|
+
let us: TimelineEntry | undefined;
|
|
226
|
+
for (let entry of entries) {
|
|
227
|
+
if (entry.pid && ancestorPids.has(entry.pid)) {
|
|
228
|
+
us = entry;
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
let computed: TakeoverComputed = {};
|
|
232
|
+
if (us) {
|
|
233
|
+
let usStart = us.aliveWindow[0];
|
|
234
|
+
let next: TimelineEntry | undefined;
|
|
235
|
+
let prev: TimelineEntry | undefined;
|
|
236
|
+
for (let entry of entries) {
|
|
237
|
+
if (entry === us) continue;
|
|
238
|
+
if (entry.aliveWindow[0] > usStart && (!next || entry.aliveWindow[0] < next.aliveWindow[0])) {
|
|
239
|
+
next = entry;
|
|
240
|
+
}
|
|
241
|
+
if (entry.aliveWindow[0] < usStart && (!prev || entry.aliveWindow[0] > prev.aliveWindow[0])) {
|
|
242
|
+
prev = entry;
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
if (next && us.aliveWindow[1] !== ALIVE_WINDOW_FOREVER && next.aliveWindow[0] <= us.aliveWindow[1]) {
|
|
246
|
+
let overlap = us.aliveWindow[1] - next.aliveWindow[0];
|
|
247
|
+
let successorStart = next.aliveWindow[0];
|
|
248
|
+
if (overlap > 0 && now < successorStart + overlap * REMAP_EXPIRE_FRACTION) {
|
|
249
|
+
computed.dying = { successorStart, overlap };
|
|
250
|
+
const successorPid = next.pid;
|
|
251
|
+
if (successorPid) {
|
|
252
|
+
let successorEntry = registry.find(x => x.ancestorPids.includes(successorPid));
|
|
253
|
+
if (successorEntry) {
|
|
254
|
+
computed.remap = makeRemap(successorStart, overlap, successorEntry.port);
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
if (prev && prev.aliveWindow[1] > usStart) {
|
|
260
|
+
let overlap = prev.aliveWindow[1] - usStart;
|
|
261
|
+
computed.predecessorEnd = prev.aliveWindow[1];
|
|
262
|
+
if (overlap > 0 && ourAltPort && now < usStart + overlap * REMAP_EXPIRE_FRACTION && !computed.remap) {
|
|
263
|
+
computed.remap = makeRemap(usStart, overlap, ourAltPort);
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
let key = JSON.stringify(computed);
|
|
268
|
+
if (key === currentKey) return;
|
|
269
|
+
current = computed;
|
|
270
|
+
currentKey = key;
|
|
271
|
+
console.log(`Deploy takeover state changed: ${key}`);
|
|
272
|
+
scheduleEvents();
|
|
273
|
+
emit("remapChanged");
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/** Starts the takeover machinery. Port fallback (alternate port + registry + acquisition polling)
|
|
277
|
+
* works regardless; without a deploy timeline folder the switchover-specific parts (the remap,
|
|
278
|
+
* the flush deadline, the tighter acquisition pacing) simply stay inert. */
|
|
279
|
+
export async function initDeployTakeover(config: { domain: string; mainPort: number; storageFolder: string }): Promise<void> {
|
|
280
|
+
initialized = config;
|
|
281
|
+
ancestorPids = await getAncestorPids();
|
|
282
|
+
timelineFolder = await findTimelineFolder();
|
|
283
|
+
if (!timelineFolder) return;
|
|
284
|
+
await recompute();
|
|
285
|
+
let poll = setInterval(() => {
|
|
286
|
+
void recompute().catch((e: Error) => console.error(`Deploy takeover recompute failed: ${e.stack ?? e}`));
|
|
287
|
+
}, TIMELINE_POLL_INTERVAL);
|
|
288
|
+
(poll as { unref?: () => void }).unref?.();
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
/** Called when we had to listen on an alternate port (the main port was still held by our
|
|
292
|
+
* predecessor): registers it so the predecessor can route the middle overlap window to us. */
|
|
293
|
+
export async function registerAltPort(port: number): Promise<void> {
|
|
294
|
+
ourAltPort = port;
|
|
295
|
+
let folder = getRegistryFolder();
|
|
296
|
+
await fs.promises.mkdir(folder, { recursive: true });
|
|
297
|
+
let entry: RegistryEntry = { pid: process.pid, port, ancestorPids: [...ancestorPids] };
|
|
298
|
+
await fs.promises.writeFile(path.join(folder, `${process.pid}.json`), JSON.stringify(entry));
|
|
299
|
+
await recompute();
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
export function onTakeoverEvent(listener: (event: TakeoverEvent) => void): void {
|
|
303
|
+
eventListeners.push(listener);
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
/** The interpretation overlay: splits every source pointing at our domain+main port so the middle
|
|
307
|
+
* of the deploy overlap points at the alternate port. Pure, in-memory only - the stored routing
|
|
308
|
+
* config is never modified, and this must never be applied to data that gets persisted. */
|
|
309
|
+
export function applyDeployRemap(routing: RemoteConfig): RemoteConfig {
|
|
310
|
+
let remap = current.remap;
|
|
311
|
+
let init = initialized;
|
|
312
|
+
if (!remap || !init) return routing;
|
|
313
|
+
let sources: RemoteConfigBase[] = [];
|
|
314
|
+
let slice = (source: HostedConfig, start: number, end: number, url?: string): HostedConfig => {
|
|
315
|
+
return { ...source, url: url || source.url, validWindow: [start, end] };
|
|
316
|
+
};
|
|
317
|
+
for (let source of routing.sources) {
|
|
318
|
+
if (typeof source === "string" || source.type !== "remote") {
|
|
319
|
+
sources.push(source);
|
|
320
|
+
continue;
|
|
321
|
+
}
|
|
322
|
+
let parsed: { address: string; port: number };
|
|
323
|
+
try {
|
|
324
|
+
parsed = parseHostedUrl(source.url);
|
|
325
|
+
} catch {
|
|
326
|
+
sources.push(source);
|
|
327
|
+
continue;
|
|
328
|
+
}
|
|
329
|
+
if (parsed.address !== init.domain || parsed.port !== init.mainPort) {
|
|
330
|
+
sources.push(source);
|
|
331
|
+
continue;
|
|
332
|
+
}
|
|
333
|
+
let [start, end] = source.validWindow;
|
|
334
|
+
let a = Math.max(start, Math.min(remap.boundaryA, end));
|
|
335
|
+
let b = Math.max(start, Math.min(remap.boundaryB, end));
|
|
336
|
+
let added = false;
|
|
337
|
+
if (a > start) {
|
|
338
|
+
sources.push(slice(source, start, a));
|
|
339
|
+
added = true;
|
|
340
|
+
}
|
|
341
|
+
if (b > a) {
|
|
342
|
+
sources.push(slice(source, a, b, replaceHostedUrlPort(source.url, remap.altPort)));
|
|
343
|
+
added = true;
|
|
344
|
+
}
|
|
345
|
+
if (end > b) {
|
|
346
|
+
sources.push(slice(source, b, end));
|
|
347
|
+
added = true;
|
|
348
|
+
}
|
|
349
|
+
if (!added) {
|
|
350
|
+
sources.push(source);
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
return { version: routing.version, sources };
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
/** For the dying process: fast-write flush delays must never extend past this time, and after it
|
|
357
|
+
* fast writes flush immediately - so nothing is left in memory when the write window transfers. */
|
|
358
|
+
export function getFlushDeadline(): number | undefined {
|
|
359
|
+
let dying = current.dying;
|
|
360
|
+
if (!dying) return undefined;
|
|
361
|
+
return dying.successorStart + dying.overlap * FLUSH_DEADLINE_FRACTION;
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
/** How long to wait between main-port acquisition attempts: tight around the predecessor's
|
|
365
|
+
* scheduled death (when the port actually frees), relaxed otherwise. */
|
|
366
|
+
export function getMainPortAcquireDelay(): number {
|
|
367
|
+
let end = current.predecessorEnd;
|
|
368
|
+
if (end !== undefined && Math.abs(Date.now() - end) <= ACQUIRE_FAST_WINDOW) {
|
|
369
|
+
return ACQUIRE_FAST_DELAY;
|
|
370
|
+
}
|
|
371
|
+
return ACQUIRE_SLOW_DELAY;
|
|
372
|
+
}
|