sliftutils 1.7.17 → 1.7.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/settings.local.json +2 -1
- package/dist/test.ts.cache +43 -8
- package/index.d.ts +5 -0
- package/misc/https/hostServer.ts +5 -1
- package/package.json +2 -2
- package/storage/remoteFileServer.ts +1 -8
- package/storage/remoteStorage/blobStore.d.ts +1 -0
- package/storage/remoteStorage/blobStore.ts +12 -0
- package/storage/remoteStorage/createArchives.ts +22 -3
- package/storage/remoteStorage/deployTakeover.ts +77 -38
- package/storage/remoteStorage/dist/createArchives.ts.cache +13 -4
- package/storage/remoteStorage/dist/sourceWrapper.ts.cache +28 -7
- package/storage/remoteStorage/sourceWrapper.d.ts +4 -0
- package/storage/remoteStorage/sourceWrapper.ts +26 -5
- package/storage/remoteStorage/storageServerState.ts +21 -8
- package/test.ts +45 -6
- package/yarn.lock +4 -4
package/dist/test.ts.cache
CHANGED
|
@@ -1,13 +1,48 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule && mod.default) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
2
5
|
Object.defineProperty(exports, "__esModule", { value: true , configurable: true});
|
|
3
|
-
const
|
|
6
|
+
const net_1 = __importDefault(require("net"));
|
|
7
|
+
const forwardPort_1 = require("socket-function/src/forwardPort");
|
|
8
|
+
const TEST_PORT = 5299;
|
|
9
|
+
const EXTERNAL_IP = "99.250.124.91";
|
|
10
|
+
function testConnect(host, port) {
|
|
11
|
+
return new Promise(resolve => {
|
|
12
|
+
let socket = net_1.default.connect({ host, port, timeout: 8000 });
|
|
13
|
+
socket.on("connect", () => {
|
|
14
|
+
socket.destroy();
|
|
15
|
+
resolve("CONNECTED");
|
|
16
|
+
});
|
|
17
|
+
socket.on("timeout", () => {
|
|
18
|
+
socket.destroy();
|
|
19
|
+
resolve("TIMEOUT");
|
|
20
|
+
});
|
|
21
|
+
socket.on("error", e => resolve(e.code || String(e)));
|
|
22
|
+
});
|
|
23
|
+
}
|
|
4
24
|
async function main() {
|
|
5
|
-
let
|
|
6
|
-
|
|
7
|
-
|
|
25
|
+
let lanIP = await (0, forwardPort_1.getLocalInternalIP)();
|
|
26
|
+
console.log("our LAN IP:", lanIP);
|
|
27
|
+
let listener = net_1.default.createServer(socket => {
|
|
28
|
+
socket.on("error", () => socket.destroy());
|
|
29
|
+
socket.end("hello");
|
|
8
30
|
});
|
|
9
|
-
|
|
31
|
+
await new Promise(resolve => listener.listen(TEST_PORT, "0.0.0.0", () => resolve()));
|
|
32
|
+
console.log(`listening on 0.0.0.0:${TEST_PORT}`);
|
|
33
|
+
await (0, forwardPort_1.forwardPort)({ externalPort: TEST_PORT, internalPort: TEST_PORT });
|
|
34
|
+
let mappings = await (0, forwardPort_1.listPortMappings)();
|
|
35
|
+
let ours = mappings.find(x => x.externalPort === TEST_PORT);
|
|
36
|
+
console.log(`mapping for ${TEST_PORT} after create:`, JSON.stringify(ours) || "MISSING");
|
|
37
|
+
console.log(`LAN direct (${lanIP}:${TEST_PORT}):`, await testConnect(lanIP || "127.0.0.1", TEST_PORT));
|
|
38
|
+
console.log(`hairpin (${EXTERNAL_IP}:${TEST_PORT}):`, await testConnect(EXTERNAL_IP, TEST_PORT));
|
|
39
|
+
listener.close();
|
|
40
|
+
process.exit(0);
|
|
10
41
|
}
|
|
11
|
-
main().catch(
|
|
12
|
-
|
|
13
|
-
|
|
42
|
+
main().catch(e => {
|
|
43
|
+
var _a;
|
|
44
|
+
console.error((_a = e.stack) !== null && _a !== void 0 ? _a : e);
|
|
45
|
+
process.exit(1);
|
|
46
|
+
});
|
|
47
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidGVzdC5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbInRlc3QudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6Ijs7Ozs7QUFBQSw4Q0FBc0I7QUFDdEIsaUVBQW9HO0FBRXBHLE1BQU0sU0FBUyxHQUFHLElBQUksQ0FBQztBQUN2QixNQUFNLFdBQVcsR0FBRyxlQUFlLENBQUM7QUFFcEMsU0FBUyxXQUFXLENBQUMsSUFBWSxFQUFFLElBQVk7SUFDM0MsT0FBTyxJQUFJLE9BQU8sQ0FBQyxPQUFPLENBQUMsRUFBRTtRQUN6QixJQUFJLE1BQU0sR0FBRyxhQUFHLENBQUMsT0FBTyxDQUFDLEVBQUUsSUFBSSxFQUFFLElBQUksRUFBRSxPQUFPLEVBQUUsSUFBSSxFQUFFLENBQUMsQ0FBQztRQUN4RCxNQUFNLENBQUMsRUFBRSxDQUFDLFNBQVMsRUFBRSxHQUFHLEVBQUU7WUFDdEIsTUFBTSxDQUFDLE9BQU8sRUFBRSxDQUFDO1lBQ2pCLE9BQU8sQ0FBQyxXQUFXLENBQUMsQ0FBQztRQUN6QixDQUFDLENBQUMsQ0FBQztRQUNILE1BQU0sQ0FBQyxFQUFFLENBQUMsU0FBUyxFQUFFLEdBQUcsRUFBRTtZQUN0QixNQUFNLENBQUMsT0FBTyxFQUFFLENBQUM7WUFDakIsT0FBTyxDQUFDLFNBQVMsQ0FBQyxDQUFDO1FBQ3ZCLENBQUMsQ0FBQyxDQUFDO1FBQ0gsTUFBTSxDQUFDLEVBQUUsQ0FBQyxPQUFPLEVBQUUsQ0FBQyxDQUFDLEVBQUUsQ0FBQyxPQUFPLENBQUUsQ0FBdUIsQ0FBQyxJQUFJLElBQUksTUFBTSxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQztJQUNqRixDQUFDLENBQUMsQ0FBQztBQUNQLENBQUM7QUFFRCxLQUFLLFVBQVUsSUFBSTtJQUNmLElBQUksS0FBSyxHQUFHLE1BQU0sSUFBQSxnQ0FBa0IsR0FBRSxDQUFDO0lBQ3ZDLE9BQU8sQ0FBQyxHQUFHLENBQUMsYUFBYSxFQUFFLEtBQUssQ0FBQyxDQUFDO0lBRWxDLElBQUksUUFBUSxHQUFHLGFBQUcsQ0FBQyxZQUFZLENBQUMsTUFBTSxDQUFDLEVBQUU7UUFDckMsTUFBTSxDQUFDLEVBQUUsQ0FBQyxPQUFPLEVBQUUsR0FBRyxFQUFFLENBQUMsTUFBTSxDQUFDLE9BQU8sRUFBRSxDQUFDLENBQUM7UUFDM0MsTUFBTSxDQUFDLEdBQUcsQ0FBQyxPQUFPLENBQUMsQ0FBQztJQUN4QixDQUFDLENBQUMsQ0FBQztJQUNILE1BQU0sSUFBSSxPQUFPLENBQU8sT0FBTyxDQUFDLEVBQUUsQ0FBQyxRQUFRLENBQUMsTUFBTSxDQUFDLFNBQVMsRUFBRSxTQUFTLEVBQUUsR0FBRyxFQUFFLENBQUMsT0FBTyxFQUFFLENBQUMsQ0FBQyxDQUFDO0lBQzNGLE9BQU8sQ0FBQyxHQUFHLENBQUMsd0JBQXdCLFNBQVMsRUFBRSxDQUFDLENBQUM7SUFFakQsTUFBTSxJQUFBLHlCQUFXLEVBQUMsRUFBRSxZQUFZLEVBQUUsU0FBUyxFQUFFLFlBQVksRUFBRSxTQUFTLEVBQUUsQ0FBQyxDQUFDO0lBRXhFLElBQUksUUFBUSxHQUFHLE1BQU0sSUFBQSw4QkFBZ0IsR0FBRSxDQUFDO0lBQ3hDLElBQUksSUFBSSxHQUFHLFFBQVEsQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQyxDQUFDLENBQUMsWUFBWSxLQUFLLFNBQVMsQ0FBQyxDQUFDO0lBQzVELE9BQU8sQ0FBQyxHQUFHLENBQUMsZUFBZSxTQUFTLGdCQUFnQixFQUFFLElBQUksQ0FBQyxTQUFTLENBQUMsSUFBSSxDQUFDLElBQUksU0FBUyxDQUFDLENBQUM7SUFFekYsT0FBTyxDQUFDLEdBQUcsQ0FBQyxlQUFlLEtBQUssSUFBSSxTQUFTLElBQUksRUFBRSxNQUFNLFdBQVcsQ0FBQyxLQUFLLElBQUksV0FBVyxFQUFFLFNBQVMsQ0FBQyxDQUFDLENBQUM7SUFDdkcsT0FBTyxDQUFDLEdBQUcsQ0FBQyxZQUFZLFdBQVcsSUFBSSxTQUFTLElBQUksRUFBRSxNQUFNLFdBQVcsQ0FBQyxXQUFXLEVBQUUsU0FBUyxDQUFDLENBQUMsQ0FBQztJQUVqRyxRQUFRLENBQUMsS0FBSyxFQUFFLENBQUM7SUFDakIsT0FBTyxDQUFDLElBQUksQ0FBQyxDQUFDLENBQUMsQ0FBQztBQUNwQixDQUFDO0FBRUQsSUFBSSxFQUFFLENBQUMsS0FBSyxDQUFDLENBQUMsQ0FBQyxFQUFFOztJQUNiLE9BQU8sQ0FBQyxLQUFLLENBQUMsTUFBQyxDQUFXLENBQUMsS0FBSyxtQ0FBSSxDQUFDLENBQUMsQ0FBQztJQUN2QyxPQUFPLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBQyxDQUFDO0FBQ3BCLENBQUMsQ0FBQyxDQUFDIiwic291cmNlc0NvbnRlbnQiOlsiaW1wb3J0IG5ldCBmcm9tIFwibmV0XCI7XG5pbXBvcnQgeyBsaXN0UG9ydE1hcHBpbmdzLCBmb3J3YXJkUG9ydCwgZ2V0TG9jYWxJbnRlcm5hbElQIH0gZnJvbSBcInNvY2tldC1mdW5jdGlvbi9zcmMvZm9yd2FyZFBvcnRcIjtcblxuY29uc3QgVEVTVF9QT1JUID0gNTI5OTtcbmNvbnN0IEVYVEVSTkFMX0lQID0gXCI5OS4yNTAuMTI0LjkxXCI7XG5cbmZ1bmN0aW9uIHRlc3RDb25uZWN0KGhvc3Q6IHN0cmluZywgcG9ydDogbnVtYmVyKTogUHJvbWlzZTxzdHJpbmc+IHtcbiAgICByZXR1cm4gbmV3IFByb21pc2UocmVzb2x2ZSA9PiB7XG4gICAgICAgIGxldCBzb2NrZXQgPSBuZXQuY29ubmVjdCh7IGhvc3QsIHBvcnQsIHRpbWVvdXQ6IDgwMDAgfSk7XG4gICAgICAgIHNvY2tldC5vbihcImNvbm5lY3RcIiwgKCkgPT4ge1xuICAgICAgICAgICAgc29ja2V0LmRlc3Ryb3koKTtcbiAgICAgICAgICAgIHJlc29sdmUoXCJDT05ORUNURURcIik7XG4gICAgICAgIH0pO1xuICAgICAgICBzb2NrZXQub24oXCJ0aW1lb3V0XCIsICgpID0+IHtcbiAgICAgICAgICAgIHNvY2tldC5kZXN0cm95KCk7XG4gICAgICAgICAgICByZXNvbHZlKFwiVElNRU9VVFwiKTtcbiAgICAgICAgfSk7XG4gICAgICAgIHNvY2tldC5vbihcImVycm9yXCIsIGUgPT4gcmVzb2x2ZSgoZSBhcyB7IGNvZGU/OiBzdHJpbmcgfSkuY29kZSB8fCBTdHJpbmcoZSkpKTtcbiAgICB9KTtcbn1cblxuYXN5bmMgZnVuY3Rpb24gbWFpbigpIHtcbiAgICBsZXQgbGFuSVAgPSBhd2FpdCBnZXRMb2NhbEludGVybmFsSVAoKTtcbiAgICBjb25zb2xlLmxvZyhcIm91ciBMQU4gSVA6XCIsIGxhbklQKTtcblxuICAgIGxldCBsaXN0ZW5lciA9IG5ldC5jcmVhdGVTZXJ2ZXIoc29ja2V0ID0+IHtcbiAgICAgICAgc29ja2V0Lm9uKFwiZXJyb3JcIiwgKCkgPT4gc29ja2V0LmRlc3Ryb3koKSk7XG4gICAgICAgIHNvY2tldC5lbmQoXCJoZWxsb1wiKTtcbiAgICB9KTtcbiAgICBhd2FpdCBuZXcgUHJvbWlzZTx2b2lkPihyZXNvbHZlID0+IGxpc3RlbmVyLmxpc3RlbihURVNUX1BPUlQsIFwiMC4wLjAuMFwiLCAoKSA9PiByZXNvbHZlKCkpKTtcbiAgICBjb25zb2xlLmxvZyhgbGlzdGVuaW5nIG9uIDAuMC4wLjA6JHtURVNUX1BPUlR9YCk7XG5cbiAgICBhd2FpdCBmb3J3YXJkUG9ydCh7IGV4dGVybmFsUG9ydDogVEVTVF9QT1JULCBpbnRlcm5hbFBvcnQ6IFRFU1RfUE9SVCB9KTtcblxuICAgIGxldCBtYXBwaW5ncyA9IGF3YWl0IGxpc3RQb3J0TWFwcGluZ3MoKTtcbiAgICBsZXQgb3VycyA9IG1hcHBpbmdzLmZpbmQoeCA9PiB4LmV4dGVybmFsUG9ydCA9PT0gVEVTVF9QT1JUKTtcbiAgICBjb25zb2xlLmxvZyhgbWFwcGluZyBmb3IgJHtURVNUX1BPUlR9IGFmdGVyIGNyZWF0ZTpgLCBKU09OLnN0cmluZ2lmeShvdXJzKSB8fCBcIk1JU1NJTkdcIik7XG5cbiAgICBjb25zb2xlLmxvZyhgTEFOIGRpcmVjdCAoJHtsYW5JUH06JHtURVNUX1BPUlR9KTpgLCBhd2FpdCB0ZXN0Q29ubmVjdChsYW5JUCB8fCBcIjEyNy4wLjAuMVwiLCBURVNUX1BPUlQpKTtcbiAgICBjb25zb2xlLmxvZyhgaGFpcnBpbiAoJHtFWFRFUk5BTF9JUH06JHtURVNUX1BPUlR9KTpgLCBhd2FpdCB0ZXN0Q29ubmVjdChFWFRFUk5BTF9JUCwgVEVTVF9QT1JUKSk7XG5cbiAgICBsaXN0ZW5lci5jbG9zZSgpO1xuICAgIHByb2Nlc3MuZXhpdCgwKTtcbn1cblxubWFpbigpLmNhdGNoKGUgPT4ge1xuICAgIGNvbnNvbGUuZXJyb3IoKGUgYXMgRXJyb3IpLnN0YWNrID8/IGUpO1xuICAgIHByb2Nlc3MuZXhpdCgxKTtcbn0pO1xuIl1//9
|
|
48
|
+
/* _JS_SOURCE_HASH = "ff635e41519c7464cb3b055e6fe334893caa4fa66ebb080597c5fd80309af3da"; */
|
package/index.d.ts
CHANGED
|
@@ -3155,6 +3155,7 @@ declare module "sliftutils/storage/remoteStorage/blobStore" {
|
|
|
3155
3155
|
cancelLargeUpload(id: string): Promise<void>;
|
|
3156
3156
|
private flushOverlay;
|
|
3157
3157
|
private evicting;
|
|
3158
|
+
private loggedFlushDeadline;
|
|
3158
3159
|
private enforceDiskLimit;
|
|
3159
3160
|
private cleanupTombstones;
|
|
3160
3161
|
}
|
|
@@ -3377,6 +3378,10 @@ declare module "sliftutils/storage/remoteStorage/sourceWrapper" {
|
|
|
3377
3378
|
/** Starts measuring this source's latency (for variable-shard target preference). Only hosted
|
|
3378
3379
|
* remotes are pinged; our own local server counts as 0, everything else as Infinity. */
|
|
3379
3380
|
startPinging(): void;
|
|
3381
|
+
/** Seeds the latency estimate before the first ping lands (e.g. from the initial routing
|
|
3382
|
+
* fetch), so variable-shard picking has something immediately. Real pings take over from the
|
|
3383
|
+
* first measurement on. */
|
|
3384
|
+
seedLatency(ms: number): void;
|
|
3380
3385
|
/** Median of the recent pings. Sources that can't be pinged sort last (Infinity), except our
|
|
3381
3386
|
* own in-process server, which is the best possible target (0). */
|
|
3382
3387
|
getLatency(): number;
|
package/misc/https/hostServer.ts
CHANGED
|
@@ -6,6 +6,7 @@ import { getNodeIdLocation } from "socket-function/src/nodeCache";
|
|
|
6
6
|
import { timeInMinute } from "socket-function/src/misc";
|
|
7
7
|
import { delay } from "socket-function/src/batching";
|
|
8
8
|
import { getExternalIP } from "socket-function/src/networking";
|
|
9
|
+
import { forwardPort } from "socket-function/src/forwardPort";
|
|
9
10
|
import { magenta } from "socket-function/src/formatting/logColors";
|
|
10
11
|
import { getOwnMachineId, getThreadKeyCert, loadIdentityCA } from "./certs";
|
|
11
12
|
import { generateCert, getAccountKey, parseCert } from "./httpsCerts";
|
|
@@ -79,7 +80,7 @@ export async function hostServer(config: HostServerConfig): Promise<string> {
|
|
|
79
80
|
cert: threadCert.cert,
|
|
80
81
|
});
|
|
81
82
|
},
|
|
82
|
-
["127-0-0-1." +
|
|
83
|
+
["127-0-0-1." + rootDomain]: async callback => {
|
|
83
84
|
callback(keyCert);
|
|
84
85
|
certListeners.push(callback);
|
|
85
86
|
},
|
|
@@ -120,6 +121,9 @@ async function runMainPortAcquireLoop(domain: string, mainPort: number, servingP
|
|
|
120
121
|
});
|
|
121
122
|
if (acquired) {
|
|
122
123
|
console.log(magenta(`Acquired main port ${mainPort} for https://${domain} (relaying to our listener on ${servingPort})`));
|
|
124
|
+
// The mount only forwarded OUR serving port; the main port needs its own mapping (the
|
|
125
|
+
// predecessor's mapping pointed at the same machine+port, but dies with it)
|
|
126
|
+
await forwardPort({ externalPort: mainPort, internalPort: mainPort });
|
|
123
127
|
fallback.onListening(mainPort, true);
|
|
124
128
|
return;
|
|
125
129
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "sliftutils",
|
|
3
|
-
"version": "1.7.
|
|
3
|
+
"version": "1.7.19",
|
|
4
4
|
"main": "index.js",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"files": [
|
|
@@ -57,7 +57,7 @@
|
|
|
57
57
|
"node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
|
|
58
58
|
"preact-old-types": "^10.28.1",
|
|
59
59
|
"shell-quote": "^1.8.3",
|
|
60
|
-
"socket-function": "^1.2.
|
|
60
|
+
"socket-function": "^1.2.24",
|
|
61
61
|
"typenode": "^6.6.1",
|
|
62
62
|
"typesafecss": "*",
|
|
63
63
|
"ws": "^8.18.3",
|
|
@@ -671,14 +671,7 @@ export async function runFileHoster(): Promise<void> {
|
|
|
671
671
|
console.log(" self-signed certificate warning, then connect from the app.");
|
|
672
672
|
console.log("");
|
|
673
673
|
|
|
674
|
-
|
|
675
|
-
// Linux (forwardPort returns early there).
|
|
676
|
-
const refresh = async () => {
|
|
677
|
-
try { await forwardPort({ externalPort: info.port, internalPort: info.port }); }
|
|
678
|
-
catch (e) { console.warn(" port forwarding failed:", (e as Error).message); }
|
|
679
|
-
};
|
|
680
|
-
await refresh();
|
|
681
|
-
setInterval(refresh, 30 * 60 * 1000);
|
|
674
|
+
await forwardPort({ externalPort: info.port, internalPort: info.port });
|
|
682
675
|
|
|
683
676
|
console.log(" [access] request logging on (batched every 5s)\n");
|
|
684
677
|
|
|
@@ -169,6 +169,7 @@ export declare class BlobStore implements IBucketStore {
|
|
|
169
169
|
cancelLargeUpload(id: string): Promise<void>;
|
|
170
170
|
private flushOverlay;
|
|
171
171
|
private evicting;
|
|
172
|
+
private loggedFlushDeadline;
|
|
172
173
|
private enforceDiskLimit;
|
|
173
174
|
private cleanupTombstones;
|
|
174
175
|
}
|
|
@@ -674,6 +674,10 @@ export class BlobStore implements IBucketStore {
|
|
|
674
674
|
let deadline = this.config?.getFlushDeadline?.();
|
|
675
675
|
if (deadline !== undefined) {
|
|
676
676
|
if (Date.now() >= deadline) {
|
|
677
|
+
if (!this.loggedFlushDeadline) {
|
|
678
|
+
this.loggedFlushDeadline = true;
|
|
679
|
+
console.log(`Deploy switchover flush deadline passed (store ${this.folder}): fast writes now write through immediately`);
|
|
680
|
+
}
|
|
677
681
|
this.overlay.delete(key);
|
|
678
682
|
await this.writeToSources(key, data, writeTime);
|
|
679
683
|
return key;
|
|
@@ -854,11 +858,14 @@ export class BlobStore implements IBucketStore {
|
|
|
854
858
|
// holds a same-or-newer copy (the only copy of a file is never deleted), and the index entry
|
|
855
859
|
// repoints to that source so reads keep working (re-caching on the next read).
|
|
856
860
|
private evicting = false;
|
|
861
|
+
private loggedFlushDeadline = false;
|
|
857
862
|
private async enforceDiskLimit(): Promise<void> {
|
|
858
863
|
let limit = this.config?.readerDiskLimit;
|
|
859
864
|
if (!limit || this.evicting) return;
|
|
860
865
|
if (this.sourceByteCounts[0] <= limit) return;
|
|
861
866
|
this.evicting = true;
|
|
867
|
+
let evictedFiles = 0;
|
|
868
|
+
let evictedBytes = 0;
|
|
862
869
|
try {
|
|
863
870
|
let candidates: { key: string; entry: IndexEntry }[] = [];
|
|
864
871
|
for (let [key, entry] of this.mem) {
|
|
@@ -885,9 +892,14 @@ export class BlobStore implements IBucketStore {
|
|
|
885
892
|
if (holder === undefined) continue;
|
|
886
893
|
await this.sources[0].source.del(key);
|
|
887
894
|
this.setIndexEntry(key, { writeTime: entry.writeTime, size: entry.size, source: holder });
|
|
895
|
+
evictedFiles++;
|
|
896
|
+
evictedBytes += entry.size;
|
|
888
897
|
}
|
|
889
898
|
} finally {
|
|
890
899
|
this.evicting = false;
|
|
900
|
+
if (evictedFiles) {
|
|
901
|
+
console.log(`Disk cache over readerDiskLimit (store ${this.folder}): evicted ${evictedFiles} least-recently-used files (${formatNumber(evictedBytes)}B), now at ${formatNumber(this.sourceByteCounts[0])}B/${formatNumber(this.config?.readerDiskLimit || 0)}B`);
|
|
902
|
+
}
|
|
891
903
|
}
|
|
892
904
|
}
|
|
893
905
|
|
|
@@ -133,11 +133,12 @@ export class ArchivesChain implements IArchives {
|
|
|
133
133
|
// that answered (the first up source is authoritative).
|
|
134
134
|
let fetches = await Promise.all(configs.map(async sourceConfig => {
|
|
135
135
|
let probe = await SourceWrapper.create(sourceConfig, { background: false });
|
|
136
|
+
let start = Date.now();
|
|
136
137
|
try {
|
|
137
138
|
let data = await probe.read(archives => archives.get(ROUTING_FILE));
|
|
138
|
-
return { probe, sourceConfig, responded: true, existing: data && parseRoutingData(data) || undefined, error: "" };
|
|
139
|
+
return { probe, sourceConfig, responded: true, latency: Date.now() - start, existing: data && parseRoutingData(data) || undefined, error: "" };
|
|
139
140
|
} catch (e) {
|
|
140
|
-
return { probe, sourceConfig, responded: false, existing: undefined, error: `${sourceConfig.url}: ${(e as Error).stack ?? e}` };
|
|
141
|
+
return { probe, sourceConfig, responded: false, latency: 0, existing: undefined, error: `${sourceConfig.url}: ${(e as Error).stack ?? e}` };
|
|
141
142
|
}
|
|
142
143
|
}));
|
|
143
144
|
try {
|
|
@@ -222,6 +223,14 @@ export class ArchivesChain implements IArchives {
|
|
|
222
223
|
}
|
|
223
224
|
}
|
|
224
225
|
}
|
|
226
|
+
// The routing fetches double as our first latency measurements: variable-shard picking
|
|
227
|
+
// works immediately, instead of waiting for the first ping pass to land
|
|
228
|
+
for (let source of sources) {
|
|
229
|
+
let fetch = fetches.find(x => x.responded && x.sourceConfig.url === source.config.url);
|
|
230
|
+
if (fetch) {
|
|
231
|
+
source.seedLatency(fetch.latency);
|
|
232
|
+
}
|
|
233
|
+
}
|
|
225
234
|
this.activeConfig = active;
|
|
226
235
|
this.startConfigPoll();
|
|
227
236
|
return { config: active, sources };
|
|
@@ -295,7 +304,13 @@ export class ArchivesChain implements IArchives {
|
|
|
295
304
|
|
|
296
305
|
private async adoptNewConfig(state: ChainState, latest: RemoteConfig): Promise<void> {
|
|
297
306
|
if (JSON.stringify(latest) === JSON.stringify(state.config)) return;
|
|
298
|
-
|
|
307
|
+
// Same version but different content is a deploy switchover remap (an in-memory window
|
|
308
|
+
// split pointing at a successor's port), not an actual configuration change
|
|
309
|
+
if (getConfigVersion(latest) === getConfigVersion(state.config)) {
|
|
310
|
+
console.log(`Storage routing reinterpreted at version ${getConfigVersion(latest)} for ${this.getDebugName()} (deploy switchover in progress), rebuilding sources. Sources: ${JSON.stringify(latest.sources)}`);
|
|
311
|
+
} else {
|
|
312
|
+
console.log(`Storage routing config changed for ${this.getDebugName()} (version ${getConfigVersion(state.config)} -> ${getConfigVersion(latest)}), rebuilding sources`);
|
|
313
|
+
}
|
|
299
314
|
let oldByConfig = new Map(state.sources.map(source => [JSON.stringify(source.config), source]));
|
|
300
315
|
let sources: SourceWrapper[] = [];
|
|
301
316
|
for (let sourceConfig of latest.sources.map(normalizeSource)) {
|
|
@@ -342,6 +357,7 @@ export class ArchivesChain implements IArchives {
|
|
|
342
357
|
// Init is failing; its own retry loop handles that
|
|
343
358
|
return;
|
|
344
359
|
}
|
|
360
|
+
console.log(`Every storage source failed for ${this.getDebugName()}; re-contacting all ${state.sources.length} sources (routing re-read + connection re-attempt)`);
|
|
345
361
|
let results = await Promise.all(state.sources.map(async source => {
|
|
346
362
|
try {
|
|
347
363
|
let data = await source.read(archives => archives.get(ROUTING_FILE));
|
|
@@ -456,12 +472,14 @@ export class ArchivesChain implements IArchives {
|
|
|
456
472
|
let now = Date.now();
|
|
457
473
|
let nearBoundary = state.sources.some(source => source.config.validWindow.some(t => t > 0 && t < Number.MAX_SAFE_INTEGER && Math.abs(t - now) <= WRONG_TARGET_BOUNDARY_WINDOW));
|
|
458
474
|
if (nearBoundary) {
|
|
475
|
+
console.log(`Write rejected by ${this.getDebugName()}: raced a valid window boundary; waiting ${WRONG_TARGET_BOUNDARY_RETRY_DELAY / 1000}s and retrying`);
|
|
459
476
|
await delay(WRONG_TARGET_BOUNDARY_RETRY_DELAY);
|
|
460
477
|
return;
|
|
461
478
|
}
|
|
462
479
|
}
|
|
463
480
|
if (Date.now() - this.lastConfigRefresh < CONFIG_REFRESH_THROTTLE) return;
|
|
464
481
|
this.lastConfigRefresh = Date.now();
|
|
482
|
+
console.log(`Write rejected by ${this.getDebugName()} (wrong ${kind}): our config is stale (likely a deploy switchover); refreshing it and retrying`);
|
|
465
483
|
await this.refreshActiveConfig();
|
|
466
484
|
}
|
|
467
485
|
|
|
@@ -665,6 +683,7 @@ export class ArchivesChain implements IArchives {
|
|
|
665
683
|
// NOTE: If we run into cases when transport level errors happen, but we are still connected, then we might want to add additional checking here, additional errors in which we will retry the other targets. However, ideally, checking if the WebSocket connection is still connected will handle all those cases. Distinguishing between application errors (which we can't retry) and transport errors.
|
|
666
684
|
if (target.isConnected()) throw e;
|
|
667
685
|
target.noteFailure();
|
|
686
|
+
console.log(`Variable-shard write target ${target.getDebugName()} is down; moving to the next-lowest-latency shard`);
|
|
668
687
|
errors.push(String((e as Error).stack ?? e));
|
|
669
688
|
}
|
|
670
689
|
}
|
|
@@ -20,7 +20,6 @@ import { parseHostedUrl, replaceHostedUrlPort } from "./remoteConfig";
|
|
|
20
20
|
// any point during a switchover.
|
|
21
21
|
|
|
22
22
|
const PARAMETERS_TIMELINE_FILE_REGEX = /^(\d+)-parameters\.json$/;
|
|
23
|
-
const ALIVE_WINDOW_FOREVER = Number.MAX_SAFE_INTEGER;
|
|
24
23
|
const TIMELINE_POLL_INTERVAL = 60 * 1000;
|
|
25
24
|
// Processes listening on an alternate port register it here (sibling of the storage folder), keyed
|
|
26
25
|
// by their actual pid; content includes their ancestor pids so the timeline entry pid (a shell
|
|
@@ -42,8 +41,10 @@ const REMAP_EXPIRE_FRACTION = 2;
|
|
|
42
41
|
|
|
43
42
|
type TimelineEntry = {
|
|
44
43
|
pid?: number;
|
|
45
|
-
|
|
46
|
-
|
|
44
|
+
// releaseTime + overlapTime define everything: the newer entry's releaseTime is when it
|
|
45
|
+
// starts, and releaseTime + overlapTime is when the older one is killed (freeing the port).
|
|
46
|
+
// The files also carry an aliveWindow, which is deliberately ignored.
|
|
47
|
+
parameters: { releaseTime?: number; overlapTime?: number };
|
|
47
48
|
};
|
|
48
49
|
|
|
49
50
|
type TakeoverComputed = {
|
|
@@ -67,6 +68,8 @@ let current: TakeoverComputed = {};
|
|
|
67
68
|
let currentKey = JSON.stringify(current);
|
|
68
69
|
let eventListeners: ((event: TakeoverEvent) => void)[] = [];
|
|
69
70
|
let eventTimers: ReturnType<typeof setTimeout>[] = [];
|
|
71
|
+
let loggedNoMatch = false;
|
|
72
|
+
let loggedIdentity = false;
|
|
70
73
|
|
|
71
74
|
function execText(command: string): Promise<string> {
|
|
72
75
|
return new Promise((resolve, reject) => {
|
|
@@ -103,20 +106,10 @@ async function getAncestorPids(): Promise<Set<number>> {
|
|
|
103
106
|
return pids;
|
|
104
107
|
}
|
|
105
108
|
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
try {
|
|
111
|
-
files = await fs.promises.readdir(dir);
|
|
112
|
-
} catch { }
|
|
113
|
-
if (files.some(file => PARAMETERS_TIMELINE_FILE_REGEX.test(file))) {
|
|
114
|
-
return dir.replaceAll("\\", "/") + "/";
|
|
115
|
-
}
|
|
116
|
-
let parent = path.dirname(dir);
|
|
117
|
-
if (parent === dir) return undefined;
|
|
118
|
-
dir = parent;
|
|
119
|
-
}
|
|
109
|
+
// The instance folder is ALWAYS the parent of our cwd (services run in <instance>/git/) - never
|
|
110
|
+
// search for it, a walk could latch onto the wrong folder
|
|
111
|
+
function getTimelineFolder(): string {
|
|
112
|
+
return path.dirname(process.cwd()).replaceAll("\\", "/") + "/";
|
|
120
113
|
}
|
|
121
114
|
|
|
122
115
|
function pidAlive(pid: number): boolean {
|
|
@@ -211,8 +204,11 @@ function scheduleEvents(): void {
|
|
|
211
204
|
};
|
|
212
205
|
let remap = current.remap;
|
|
213
206
|
if (remap) {
|
|
207
|
+
let altPort = remap.altPort;
|
|
214
208
|
schedule(remap.boundaryA - SWITCH_SCAN_LEAD, () => emit("diskScan"));
|
|
209
|
+
schedule(remap.boundaryA, () => console.log(`Deploy switchover write handoff boundary passed: fresh writes now belong to the alternate-port side (port ${altPort})`));
|
|
215
210
|
schedule(remap.boundaryA + SWITCH_SCAN_LEAD, () => emit("diskScan"));
|
|
211
|
+
schedule(remap.boundaryB, () => console.log(`Deploy switchover second boundary passed: fresh writes return to the main port`));
|
|
216
212
|
schedule(remap.expire, () => void recompute().catch((e: Error) => console.error(`Deploy takeover recompute failed: ${e.stack ?? e}`)));
|
|
217
213
|
}
|
|
218
214
|
}
|
|
@@ -228,26 +224,42 @@ async function recompute(): Promise<void> {
|
|
|
228
224
|
us = entry;
|
|
229
225
|
}
|
|
230
226
|
}
|
|
227
|
+
// Matching failures must be loud, or takeover detection fails silently forever
|
|
228
|
+
if (!us && entries.length && !loggedNoMatch) {
|
|
229
|
+
loggedNoMatch = true;
|
|
230
|
+
console.warn(`No deploy timeline entry matches our pid chain (${[...ancestorPids].join(", ")}) - takeover detection cannot work. Entries: ${entries.map(x => `pid=${x.pid ?? "none"} releaseTime=${x.parameters.releaseTime}`).join(" | ")}`);
|
|
231
|
+
}
|
|
232
|
+
if (us && !loggedIdentity) {
|
|
233
|
+
loggedIdentity = true;
|
|
234
|
+
console.log(`Deploy timeline entry matched: we are pid=${us.pid}, releaseTime=${us.parameters.releaseTime !== undefined && iso(us.parameters.releaseTime) || "unknown"}, ${entries.length} entries total`);
|
|
235
|
+
}
|
|
231
236
|
let computed: TakeoverComputed = {};
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
237
|
+
let usRelease = us?.parameters.releaseTime;
|
|
238
|
+
if (us && usRelease !== undefined) {
|
|
239
|
+
// The newer entry's releaseTime is when it starts; releaseTime + overlapTime is when the
|
|
240
|
+
// older one is killed. That's the whole timeline.
|
|
241
|
+
let next: { entry: TimelineEntry; release: number } | undefined;
|
|
242
|
+
let prev: { entry: TimelineEntry; release: number } | undefined;
|
|
236
243
|
for (let entry of entries) {
|
|
237
244
|
if (entry === us) continue;
|
|
238
|
-
|
|
239
|
-
|
|
245
|
+
let release = entry.parameters.releaseTime;
|
|
246
|
+
if (release === undefined) continue;
|
|
247
|
+
if (release > usRelease && (!next || release < next.release)) {
|
|
248
|
+
next = { entry, release };
|
|
240
249
|
}
|
|
241
|
-
if (
|
|
242
|
-
prev = entry;
|
|
250
|
+
if (release < usRelease && (!prev || release > prev.release)) {
|
|
251
|
+
prev = { entry, release };
|
|
243
252
|
}
|
|
244
253
|
}
|
|
245
|
-
if (next
|
|
246
|
-
let
|
|
247
|
-
let
|
|
254
|
+
if (next) {
|
|
255
|
+
let successorStart = next.release;
|
|
256
|
+
let overlap = next.entry.parameters.overlapTime || us.parameters.overlapTime || 0;
|
|
257
|
+
// Even a ZERO-overlap deploy needs the dying behaviors - draining fast-write flushes
|
|
258
|
+
// by the release and writing through after it, or acknowledged data dies with us.
|
|
259
|
+
// Only the remap (the alternate-port middle window) needs an actual overlap.
|
|
260
|
+
computed.dying = { successorStart, overlap };
|
|
248
261
|
if (overlap > 0 && now < successorStart + overlap * REMAP_EXPIRE_FRACTION) {
|
|
249
|
-
|
|
250
|
-
const successorPid = next.pid;
|
|
262
|
+
const successorPid = next.entry.pid;
|
|
251
263
|
if (successorPid) {
|
|
252
264
|
let successorEntry = registry.find(x => x.ancestorPids.includes(successorPid));
|
|
253
265
|
if (successorEntry) {
|
|
@@ -256,31 +268,58 @@ async function recompute(): Promise<void> {
|
|
|
256
268
|
}
|
|
257
269
|
}
|
|
258
270
|
}
|
|
259
|
-
if (prev
|
|
260
|
-
let overlap = prev.
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
271
|
+
if (prev) {
|
|
272
|
+
let overlap = us.parameters.overlapTime || prev.entry.parameters.overlapTime || 0;
|
|
273
|
+
// The predecessor holds the main port until OUR release + the overlap
|
|
274
|
+
computed.predecessorEnd = usRelease + overlap;
|
|
275
|
+
if (overlap > 0 && ourAltPort && now < usRelease + overlap * REMAP_EXPIRE_FRACTION && !computed.remap) {
|
|
276
|
+
computed.remap = makeRemap(usRelease, overlap, ourAltPort);
|
|
264
277
|
}
|
|
265
278
|
}
|
|
266
279
|
}
|
|
267
280
|
let key = JSON.stringify(computed);
|
|
268
281
|
if (key === currentKey) return;
|
|
282
|
+
logTransitions(current, computed);
|
|
269
283
|
current = computed;
|
|
270
284
|
currentKey = key;
|
|
271
|
-
console.log(`Deploy takeover state changed: ${key}`);
|
|
272
285
|
scheduleEvents();
|
|
273
286
|
emit("remapChanged");
|
|
274
287
|
}
|
|
275
288
|
|
|
289
|
+
function iso(time: number): string {
|
|
290
|
+
return new Date(time).toISOString();
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
// The switchover lifecycle is rare and important, so every stage gets a clear line
|
|
294
|
+
function logTransitions(prev: TakeoverComputed, next: TakeoverComputed): void {
|
|
295
|
+
if (!prev.dying && next.dying) {
|
|
296
|
+
let { successorStart, overlap } = next.dying;
|
|
297
|
+
console.log(`Deploy switchover scheduled: our successor starts at ${iso(successorStart)}; fast-write flushes drain by ${iso(successorStart + overlap * FLUSH_DEADLINE_FRACTION)}, fresh writes hand off at ${iso(successorStart + overlap * BOUNDARY_A_FRACTION)}, and we are killed at ${iso(successorStart + overlap)}`);
|
|
298
|
+
}
|
|
299
|
+
if (!prev.predecessorEnd && next.predecessorEnd) {
|
|
300
|
+
console.log(`We are a deploy successor: our predecessor holds the main port until ${iso(next.predecessorEnd)}`);
|
|
301
|
+
}
|
|
302
|
+
if (!prev.remap && next.remap) {
|
|
303
|
+
console.log(`Deploy switchover remap active: sources pointing at us are split so [${iso(next.remap.boundaryA)} .. ${iso(next.remap.boundaryB)}] routes to alternate port ${next.remap.altPort} (remap expires at ${iso(next.remap.expire)})`);
|
|
304
|
+
}
|
|
305
|
+
if (prev.remap && !next.remap) {
|
|
306
|
+
console.log(`Deploy switchover remap ended; running on the plain stored routing config`);
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
|
|
276
310
|
/** Starts the takeover machinery. Port fallback (alternate port + registry + acquisition polling)
|
|
277
311
|
* works regardless; without a deploy timeline folder the switchover-specific parts (the remap,
|
|
278
312
|
* the flush deadline, the tighter acquisition pacing) simply stay inert. */
|
|
279
313
|
export async function initDeployTakeover(config: { domain: string; mainPort: number; storageFolder: string }): Promise<void> {
|
|
280
314
|
initialized = config;
|
|
281
315
|
ancestorPids = await getAncestorPids();
|
|
282
|
-
timelineFolder =
|
|
283
|
-
|
|
316
|
+
timelineFolder = getTimelineFolder();
|
|
317
|
+
let entries = await readTimelineEntries();
|
|
318
|
+
if (!entries.length) {
|
|
319
|
+
console.log(`No deploy timeline entries in ${timelineFolder}; deploy takeover inactive until they appear (port fallback still works)`);
|
|
320
|
+
} else {
|
|
321
|
+
console.log(`Deploy timeline found at ${timelineFolder} (${entries.length} entries); our pid chain: ${[...ancestorPids].join(" -> ")}`);
|
|
322
|
+
}
|
|
284
323
|
await recompute();
|
|
285
324
|
let poll = setInterval(() => {
|
|
286
325
|
void recompute().catch((e: Error) => console.error(`Deploy takeover recompute failed: ${e.stack ?? e}`));
|