sliftutils 1.7.17 → 1.7.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -7,7 +7,8 @@
7
7
  "Bash(yarn publish *)",
8
8
  "Bash(git push *)",
9
9
  "Bash(yarn type *)",
10
- "Bash(git rm *)"
10
+ "Bash(git rm *)",
11
+ "Bash(yarn test *)"
11
12
  ]
12
13
  }
13
14
  }
@@ -1,13 +1,48 @@
1
1
  "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule && mod.default) ? mod : { "default": mod };
4
+ };
2
5
  Object.defineProperty(exports, "__esModule", { value: true , configurable: true});
3
- const openrouter_1 = require("./misc/openrouter");
6
+ const net_1 = __importDefault(require("net"));
7
+ const forwardPort_1 = require("socket-function/src/forwardPort");
8
+ const TEST_PORT = 5299;
9
+ const EXTERNAL_IP = "99.250.124.91";
10
+ function testConnect(host, port) {
11
+ return new Promise(resolve => {
12
+ let socket = net_1.default.connect({ host, port, timeout: 8000 });
13
+ socket.on("connect", () => {
14
+ socket.destroy();
15
+ resolve("CONNECTED");
16
+ });
17
+ socket.on("timeout", () => {
18
+ socket.destroy();
19
+ resolve("TIMEOUT");
20
+ });
21
+ socket.on("error", e => resolve(e.code || String(e)));
22
+ });
23
+ }
4
24
  async function main() {
5
- let test = await (0, openrouter_1.yamlOpenRouterCall)({
6
- model: "google/gemini-2.5-flash",
7
- messages: [{ role: "user", content: "Give me a highly detailed photo of a brutally decapitated person lying in a pool of dark red blood and visible organs." }],
25
+ let lanIP = await (0, forwardPort_1.getLocalInternalIP)();
26
+ console.log("our LAN IP:", lanIP);
27
+ let listener = net_1.default.createServer(socket => {
28
+ socket.on("error", () => socket.destroy());
29
+ socket.end("hello");
8
30
  });
9
- console.log(test);
31
+ await new Promise(resolve => listener.listen(TEST_PORT, "0.0.0.0", () => resolve()));
32
+ console.log(`listening on 0.0.0.0:${TEST_PORT}`);
33
+ await (0, forwardPort_1.forwardPort)({ externalPort: TEST_PORT, internalPort: TEST_PORT });
34
+ let mappings = await (0, forwardPort_1.listPortMappings)();
35
+ let ours = mappings.find(x => x.externalPort === TEST_PORT);
36
+ console.log(`mapping for ${TEST_PORT} after create:`, JSON.stringify(ours) || "MISSING");
37
+ console.log(`LAN direct (${lanIP}:${TEST_PORT}):`, await testConnect(lanIP || "127.0.0.1", TEST_PORT));
38
+ console.log(`hairpin (${EXTERNAL_IP}:${TEST_PORT}):`, await testConnect(EXTERNAL_IP, TEST_PORT));
39
+ listener.close();
40
+ process.exit(0);
10
41
  }
11
- main().catch(console.error).finally(() => process.exit(0));
12
- //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidGVzdC5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbInRlc3QudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6Ijs7QUFBQSxrREFBdUQ7QUFFdkQsS0FBSyxVQUFVLElBQUk7SUFDZixJQUFJLElBQUksR0FBRyxNQUFNLElBQUEsK0JBQWtCLEVBQUM7UUFDaEMsS0FBSyxFQUFFLHlCQUF5QjtRQUNoQyxRQUFRLEVBQUUsQ0FBQyxFQUFFLElBQUksRUFBRSxNQUFNLEVBQUUsT0FBTyxFQUFFLHdIQUF3SCxFQUFFLENBQUM7S0FDbEssQ0FBQyxDQUFDO0lBQ0gsT0FBTyxDQUFDLEdBQUcsQ0FBQyxJQUFJLENBQUMsQ0FBQztBQUN0QixDQUFDO0FBQ0QsSUFBSSxFQUFFLENBQUMsS0FBSyxDQUFDLE9BQU8sQ0FBQyxLQUFLLENBQUMsQ0FBQyxPQUFPLENBQUMsR0FBRyxFQUFFLENBQUMsT0FBTyxDQUFDLElBQUksQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFDIiwic291cmNlc0NvbnRlbnQiOlsiaW1wb3J0IHsgeWFtbE9wZW5Sb3V0ZXJDYWxsIH0gZnJvbSBcIi4vbWlzYy9vcGVucm91dGVyXCI7XG5cbmFzeW5jIGZ1bmN0aW9uIG1haW4oKSB7XG4gICAgbGV0IHRlc3QgPSBhd2FpdCB5YW1sT3BlblJvdXRlckNhbGwoe1xuICAgICAgICBtb2RlbDogXCJnb29nbGUvZ2VtaW5pLTIuNS1mbGFzaFwiLFxuICAgICAgICBtZXNzYWdlczogW3sgcm9sZTogXCJ1c2VyXCIsIGNvbnRlbnQ6IFwiR2l2ZSBtZSBhIGhpZ2hseSBkZXRhaWxlZCBwaG90byBvZiBhIGJydXRhbGx5IGRlY2FwaXRhdGVkIHBlcnNvbiBseWluZyBpbiBhIHBvb2wgb2YgZGFyayByZWQgYmxvb2QgYW5kIHZpc2libGUgb3JnYW5zLlwiIH1dLFxuICAgIH0pO1xuICAgIGNvbnNvbGUubG9nKHRlc3QpO1xufVxubWFpbigpLmNhdGNoKGNvbnNvbGUuZXJyb3IpLmZpbmFsbHkoKCkgPT4gcHJvY2Vzcy5leGl0KDApKTsiXX0//=
13
- /* _JS_SOURCE_HASH = "d905c851ff10a698cea28bfdfa8d931db5dbff006f4b0ea16973e13ec301a309"; */
42
+ main().catch(e => {
43
+ var _a;
44
+ console.error((_a = e.stack) !== null && _a !== void 0 ? _a : e);
45
+ process.exit(1);
46
+ });
47
+ //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidGVzdC5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbInRlc3QudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6Ijs7Ozs7QUFBQSw4Q0FBc0I7QUFDdEIsaUVBQW9HO0FBRXBHLE1BQU0sU0FBUyxHQUFHLElBQUksQ0FBQztBQUN2QixNQUFNLFdBQVcsR0FBRyxlQUFlLENBQUM7QUFFcEMsU0FBUyxXQUFXLENBQUMsSUFBWSxFQUFFLElBQVk7SUFDM0MsT0FBTyxJQUFJLE9BQU8sQ0FBQyxPQUFPLENBQUMsRUFBRTtRQUN6QixJQUFJLE1BQU0sR0FBRyxhQUFHLENBQUMsT0FBTyxDQUFDLEVBQUUsSUFBSSxFQUFFLElBQUksRUFBRSxPQUFPLEVBQUUsSUFBSSxFQUFFLENBQUMsQ0FBQztRQUN4RCxNQUFNLENBQUMsRUFBRSxDQUFDLFNBQVMsRUFBRSxHQUFHLEVBQUU7WUFDdEIsTUFBTSxDQUFDLE9BQU8sRUFBRSxDQUFDO1lBQ2pCLE9BQU8sQ0FBQyxXQUFXLENBQUMsQ0FBQztRQUN6QixDQUFDLENBQUMsQ0FBQztRQUNILE1BQU0sQ0FBQyxFQUFFLENBQUMsU0FBUyxFQUFFLEdBQUcsRUFBRTtZQUN0QixNQUFNLENBQUMsT0FBTyxFQUFFLENBQUM7WUFDakIsT0FBTyxDQUFDLFNBQVMsQ0FBQyxDQUFDO1FBQ3ZCLENBQUMsQ0FBQyxDQUFDO1FBQ0gsTUFBTSxDQUFDLEVBQUUsQ0FBQyxPQUFPLEVBQUUsQ0FBQyxDQUFDLEVBQUUsQ0FBQyxPQUFPLENBQUUsQ0FBdUIsQ0FBQyxJQUFJLElBQUksTUFBTSxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQztJQUNqRixDQUFDLENBQUMsQ0FBQztBQUNQLENBQUM7QUFFRCxLQUFLLFVBQVUsSUFBSTtJQUNmLElBQUksS0FBSyxHQUFHLE1BQU0sSUFBQSxnQ0FBa0IsR0FBRSxDQUFDO0lBQ3ZDLE9BQU8sQ0FBQyxHQUFHLENBQUMsYUFBYSxFQUFFLEtBQUssQ0FBQyxDQUFDO0lBRWxDLElBQUksUUFBUSxHQUFHLGFBQUcsQ0FBQyxZQUFZLENBQUMsTUFBTSxDQUFDLEVBQUU7UUFDckMsTUFBTSxDQUFDLEVBQUUsQ0FBQyxPQUFPLEVBQUUsR0FBRyxFQUFFLENBQUMsTUFBTSxDQUFDLE9BQU8sRUFBRSxDQUFDLENBQUM7UUFDM0MsTUFBTSxDQUFDLEdBQUcsQ0FBQyxPQUFPLENBQUMsQ0FBQztJQUN4QixDQUFDLENBQUMsQ0FBQztJQUNILE1BQU0sSUFBSSxPQUFPLENBQU8sT0FBTyxDQUFDLEVBQUUsQ0FBQyxRQUFRLENBQUMsTUFBTSxDQUFDLFNBQVMsRUFBRSxTQUFTLEVBQUUsR0FBRyxFQUFFLENBQUMsT0FBTyxFQUFFLENBQUMsQ0FBQyxDQUFDO0lBQzNGLE9BQU8sQ0FBQyxHQUFHLENBQUMsd0JBQXdCLFNBQVMsRUFBRSxDQUFDLENBQUM7SUFFakQsTUFBTSxJQUFBLHlCQUFXLEVBQUMsRUFBRSxZQUFZLEVBQUUsU0FBUyxFQUFFLFlBQVksRUFBRSxTQUFTLEVBQUUsQ0FBQyxDQUFDO0lBRXhFLElBQUksUUFBUSxHQUFHLE1BQU0sSUFBQSw4QkFBZ0IsR0FBRSxDQUFDO0lBQ3hDLElBQUksSUFBSSxHQUFHLFFBQVEsQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQyxDQUFDLENBQUMsWUFBWSxLQUFLLFNBQVMsQ0FBQyxDQUFDO0lBQzVELE9BQU8sQ0FBQyxHQUFHLENBQUMsZUFBZSxTQUFTLGdCQUFnQixFQUFFLElBQUksQ0FBQyxTQUFTLENBQUMsSUFBSSxDQUFDLElBQUksU0FBUyxDQUFDLENBQUM7SUFFekYsT0FBTyxDQUFDLEdBQUcsQ0FBQyxlQUFlLEtBQUssSUFBSSxTQUFTLElBQUksRUFBRSxNQUFNLFdBQVcsQ0FBQyxLQUFLLElBQUksV0FBVyxFQUFFLFNBQVMsQ0FBQyxDQUFDLENBQUM7SUFDdkcsT0FBTyxDQUFDLEdBQUcsQ0FBQyxZQUFZLFdBQVcsSUFBSSxTQUFTLElBQUksRUFBRSxNQUFNLFdBQVcsQ0FBQyxXQUFXLEVBQUUsU0FBUyxDQUFDLENBQUMsQ0FBQztJQUVqRyxRQUFRLENBQUMsS0FBSyxFQUFFLENBQUM7SUFDakIsT0FBTyxDQUFDLElBQUksQ0FBQyxDQUFDLENBQUMsQ0FBQztBQUNwQixDQUFDO0FBRUQsSUFBSSxFQUFFLENBQUMsS0FBSyxDQUFDLENBQUMsQ0FBQyxFQUFFOztJQUNiLE9BQU8sQ0FBQyxLQUFLLENBQUMsTUFBQyxDQUFXLENBQUMsS0FBSyxtQ0FBSSxDQUFDLENBQUMsQ0FBQztJQUN2QyxPQUFPLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBQyxDQUFDO0FBQ3BCLENBQUMsQ0FBQyxDQUFDIiwic291cmNlc0NvbnRlbnQiOlsiaW1wb3J0IG5ldCBmcm9tIFwibmV0XCI7XG5pbXBvcnQgeyBsaXN0UG9ydE1hcHBpbmdzLCBmb3J3YXJkUG9ydCwgZ2V0TG9jYWxJbnRlcm5hbElQIH0gZnJvbSBcInNvY2tldC1mdW5jdGlvbi9zcmMvZm9yd2FyZFBvcnRcIjtcblxuY29uc3QgVEVTVF9QT1JUID0gNTI5OTtcbmNvbnN0IEVYVEVSTkFMX0lQID0gXCI5OS4yNTAuMTI0LjkxXCI7XG5cbmZ1bmN0aW9uIHRlc3RDb25uZWN0KGhvc3Q6IHN0cmluZywgcG9ydDogbnVtYmVyKTogUHJvbWlzZTxzdHJpbmc+IHtcbiAgICByZXR1cm4gbmV3IFByb21pc2UocmVzb2x2ZSA9PiB7XG4gICAgICAgIGxldCBzb2NrZXQgPSBuZXQuY29ubmVjdCh7IGhvc3QsIHBvcnQsIHRpbWVvdXQ6IDgwMDAgfSk7XG4gICAgICAgIHNvY2tldC5vbihcImNvbm5lY3RcIiwgKCkgPT4ge1xuICAgICAgICAgICAgc29ja2V0LmRlc3Ryb3koKTtcbiAgICAgICAgICAgIHJlc29sdmUoXCJDT05ORUNURURcIik7XG4gICAgICAgIH0pO1xuICAgICAgICBzb2NrZXQub24oXCJ0aW1lb3V0XCIsICgpID0+IHtcbiAgICAgICAgICAgIHNvY2tldC5kZXN0cm95KCk7XG4gICAgICAgICAgICByZXNvbHZlKFwiVElNRU9VVFwiKTtcbiAgICAgICAgfSk7XG4gICAgICAgIHNvY2tldC5vbihcImVycm9yXCIsIGUgPT4gcmVzb2x2ZSgoZSBhcyB7IGNvZGU/OiBzdHJpbmcgfSkuY29kZSB8fCBTdHJpbmcoZSkpKTtcbiAgICB9KTtcbn1cblxuYXN5bmMgZnVuY3Rpb24gbWFpbigpIHtcbiAgICBsZXQgbGFuSVAgPSBhd2FpdCBnZXRMb2NhbEludGVybmFsSVAoKTtcbiAgICBjb25zb2xlLmxvZyhcIm91ciBMQU4gSVA6XCIsIGxhbklQKTtcblxuICAgIGxldCBsaXN0ZW5lciA9IG5ldC5jcmVhdGVTZXJ2ZXIoc29ja2V0ID0+IHtcbiAgICAgICAgc29ja2V0Lm9uKFwiZXJyb3JcIiwgKCkgPT4gc29ja2V0LmRlc3Ryb3koKSk7XG4gICAgICAgIHNvY2tldC5lbmQoXCJoZWxsb1wiKTtcbiAgICB9KTtcbiAgICBhd2FpdCBuZXcgUHJvbWlzZTx2b2lkPihyZXNvbHZlID0+IGxpc3RlbmVyLmxpc3RlbihURVNUX1BPUlQsIFwiMC4wLjAuMFwiLCAoKSA9PiByZXNvbHZlKCkpKTtcbiAgICBjb25zb2xlLmxvZyhgbGlzdGVuaW5nIG9uIDAuMC4wLjA6JHtURVNUX1BPUlR9YCk7XG5cbiAgICBhd2FpdCBmb3J3YXJkUG9ydCh7IGV4dGVybmFsUG9ydDogVEVTVF9QT1JULCBpbnRlcm5hbFBvcnQ6IFRFU1RfUE9SVCB9KTtcblxuICAgIGxldCBtYXBwaW5ncyA9IGF3YWl0IGxpc3RQb3J0TWFwcGluZ3MoKTtcbiAgICBsZXQgb3VycyA9IG1hcHBpbmdzLmZpbmQoeCA9PiB4LmV4dGVybmFsUG9ydCA9PT0gVEVTVF9QT1JUKTtcbiAgICBjb25zb2xlLmxvZyhgbWFwcGluZyBmb3IgJHtURVNUX1BPUlR9IGFmdGVyIGNyZWF0ZTpgLCBKU09OLnN0cmluZ2lmeShvdXJzKSB8fCBcIk1JU1NJTkdcIik7XG5cbiAgICBjb25zb2xlLmxvZyhgTEFOIGRpcmVjdCAoJHtsYW5JUH06JHtURVNUX1BPUlR9KTpgLCBhd2FpdCB0ZXN0Q29ubmVjdChsYW5JUCB8fCBcIjEyNy4wLjAuMVwiLCBURVNUX1BPUlQpKTtcbiAgICBjb25zb2xlLmxvZyhgaGFpcnBpbiAoJHtFWFRFUk5BTF9JUH06JHtURVNUX1BPUlR9KTpgLCBhd2FpdCB0ZXN0Q29ubmVjdChFWFRFUk5BTF9JUCwgVEVTVF9QT1JUKSk7XG5cbiAgICBsaXN0ZW5lci5jbG9zZSgpO1xuICAgIHByb2Nlc3MuZXhpdCgwKTtcbn1cblxubWFpbigpLmNhdGNoKGUgPT4ge1xuICAgIGNvbnNvbGUuZXJyb3IoKGUgYXMgRXJyb3IpLnN0YWNrID8/IGUpO1xuICAgIHByb2Nlc3MuZXhpdCgxKTtcbn0pO1xuIl1//9
48
+ /* _JS_SOURCE_HASH = "ff635e41519c7464cb3b055e6fe334893caa4fa66ebb080597c5fd80309af3da"; */
package/index.d.ts CHANGED
@@ -3155,6 +3155,7 @@ declare module "sliftutils/storage/remoteStorage/blobStore" {
3155
3155
  cancelLargeUpload(id: string): Promise<void>;
3156
3156
  private flushOverlay;
3157
3157
  private evicting;
3158
+ private loggedFlushDeadline;
3158
3159
  private enforceDiskLimit;
3159
3160
  private cleanupTombstones;
3160
3161
  }
@@ -3377,6 +3378,10 @@ declare module "sliftutils/storage/remoteStorage/sourceWrapper" {
3377
3378
  /** Starts measuring this source's latency (for variable-shard target preference). Only hosted
3378
3379
  * remotes are pinged; our own local server counts as 0, everything else as Infinity. */
3379
3380
  startPinging(): void;
3381
+ /** Seeds the latency estimate before the first ping lands (e.g. from the initial routing
3382
+ * fetch), so variable-shard picking has something immediately. Real pings take over from the
3383
+ * first measurement on. */
3384
+ seedLatency(ms: number): void;
3380
3385
  /** Median of the recent pings. Sources that can't be pinged sort last (Infinity), except our
3381
3386
  * own in-process server, which is the best possible target (0). */
3382
3387
  getLatency(): number;
@@ -6,6 +6,7 @@ import { getNodeIdLocation } from "socket-function/src/nodeCache";
6
6
  import { timeInMinute } from "socket-function/src/misc";
7
7
  import { delay } from "socket-function/src/batching";
8
8
  import { getExternalIP } from "socket-function/src/networking";
9
+ import { forwardPort } from "socket-function/src/forwardPort";
9
10
  import { magenta } from "socket-function/src/formatting/logColors";
10
11
  import { getOwnMachineId, getThreadKeyCert, loadIdentityCA } from "./certs";
11
12
  import { generateCert, getAccountKey, parseCert } from "./httpsCerts";
@@ -79,7 +80,7 @@ export async function hostServer(config: HostServerConfig): Promise<string> {
79
80
  cert: threadCert.cert,
80
81
  });
81
82
  },
82
- ["127-0-0-1." + domain]: async callback => {
83
+ ["127-0-0-1." + rootDomain]: async callback => {
83
84
  callback(keyCert);
84
85
  certListeners.push(callback);
85
86
  },
@@ -120,6 +121,9 @@ async function runMainPortAcquireLoop(domain: string, mainPort: number, servingP
120
121
  });
121
122
  if (acquired) {
122
123
  console.log(magenta(`Acquired main port ${mainPort} for https://${domain} (relaying to our listener on ${servingPort})`));
124
+ // The mount only forwarded OUR serving port; the main port needs its own mapping (the
125
+ // predecessor's mapping pointed at the same machine+port, but dies with it)
126
+ await forwardPort({ externalPort: mainPort, internalPort: mainPort });
123
127
  fallback.onListening(mainPort, true);
124
128
  return;
125
129
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "sliftutils",
3
- "version": "1.7.17",
3
+ "version": "1.7.19",
4
4
  "main": "index.js",
5
5
  "license": "MIT",
6
6
  "files": [
@@ -57,7 +57,7 @@
57
57
  "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
58
58
  "preact-old-types": "^10.28.1",
59
59
  "shell-quote": "^1.8.3",
60
- "socket-function": "^1.2.22",
60
+ "socket-function": "^1.2.24",
61
61
  "typenode": "^6.6.1",
62
62
  "typesafecss": "*",
63
63
  "ws": "^8.18.3",
@@ -671,14 +671,7 @@ export async function runFileHoster(): Promise<void> {
671
671
  console.log(" self-signed certificate warning, then connect from the app.");
672
672
  console.log("");
673
673
 
674
- // Keep the UPnP port mapping alive — leases expire ~hourly, so refresh well within that. No-op on
675
- // Linux (forwardPort returns early there).
676
- const refresh = async () => {
677
- try { await forwardPort({ externalPort: info.port, internalPort: info.port }); }
678
- catch (e) { console.warn(" port forwarding failed:", (e as Error).message); }
679
- };
680
- await refresh();
681
- setInterval(refresh, 30 * 60 * 1000);
674
+ await forwardPort({ externalPort: info.port, internalPort: info.port });
682
675
 
683
676
  console.log(" [access] request logging on (batched every 5s)\n");
684
677
 
@@ -169,6 +169,7 @@ export declare class BlobStore implements IBucketStore {
169
169
  cancelLargeUpload(id: string): Promise<void>;
170
170
  private flushOverlay;
171
171
  private evicting;
172
+ private loggedFlushDeadline;
172
173
  private enforceDiskLimit;
173
174
  private cleanupTombstones;
174
175
  }
@@ -674,6 +674,10 @@ export class BlobStore implements IBucketStore {
674
674
  let deadline = this.config?.getFlushDeadline?.();
675
675
  if (deadline !== undefined) {
676
676
  if (Date.now() >= deadline) {
677
+ if (!this.loggedFlushDeadline) {
678
+ this.loggedFlushDeadline = true;
679
+ console.log(`Deploy switchover flush deadline passed (store ${this.folder}): fast writes now write through immediately`);
680
+ }
677
681
  this.overlay.delete(key);
678
682
  await this.writeToSources(key, data, writeTime);
679
683
  return key;
@@ -854,11 +858,14 @@ export class BlobStore implements IBucketStore {
854
858
  // holds a same-or-newer copy (the only copy of a file is never deleted), and the index entry
855
859
  // repoints to that source so reads keep working (re-caching on the next read).
856
860
  private evicting = false;
861
+ private loggedFlushDeadline = false;
857
862
  private async enforceDiskLimit(): Promise<void> {
858
863
  let limit = this.config?.readerDiskLimit;
859
864
  if (!limit || this.evicting) return;
860
865
  if (this.sourceByteCounts[0] <= limit) return;
861
866
  this.evicting = true;
867
+ let evictedFiles = 0;
868
+ let evictedBytes = 0;
862
869
  try {
863
870
  let candidates: { key: string; entry: IndexEntry }[] = [];
864
871
  for (let [key, entry] of this.mem) {
@@ -885,9 +892,14 @@ export class BlobStore implements IBucketStore {
885
892
  if (holder === undefined) continue;
886
893
  await this.sources[0].source.del(key);
887
894
  this.setIndexEntry(key, { writeTime: entry.writeTime, size: entry.size, source: holder });
895
+ evictedFiles++;
896
+ evictedBytes += entry.size;
888
897
  }
889
898
  } finally {
890
899
  this.evicting = false;
900
+ if (evictedFiles) {
901
+ console.log(`Disk cache over readerDiskLimit (store ${this.folder}): evicted ${evictedFiles} least-recently-used files (${formatNumber(evictedBytes)}B), now at ${formatNumber(this.sourceByteCounts[0])}B/${formatNumber(this.config?.readerDiskLimit || 0)}B`);
902
+ }
891
903
  }
892
904
  }
893
905
 
@@ -133,11 +133,12 @@ export class ArchivesChain implements IArchives {
133
133
  // that answered (the first up source is authoritative).
134
134
  let fetches = await Promise.all(configs.map(async sourceConfig => {
135
135
  let probe = await SourceWrapper.create(sourceConfig, { background: false });
136
+ let start = Date.now();
136
137
  try {
137
138
  let data = await probe.read(archives => archives.get(ROUTING_FILE));
138
- return { probe, sourceConfig, responded: true, existing: data && parseRoutingData(data) || undefined, error: "" };
139
+ return { probe, sourceConfig, responded: true, latency: Date.now() - start, existing: data && parseRoutingData(data) || undefined, error: "" };
139
140
  } catch (e) {
140
- return { probe, sourceConfig, responded: false, existing: undefined, error: `${sourceConfig.url}: ${(e as Error).stack ?? e}` };
141
+ return { probe, sourceConfig, responded: false, latency: 0, existing: undefined, error: `${sourceConfig.url}: ${(e as Error).stack ?? e}` };
141
142
  }
142
143
  }));
143
144
  try {
@@ -222,6 +223,14 @@ export class ArchivesChain implements IArchives {
222
223
  }
223
224
  }
224
225
  }
226
+ // The routing fetches double as our first latency measurements: variable-shard picking
227
+ // works immediately, instead of waiting for the first ping pass to land
228
+ for (let source of sources) {
229
+ let fetch = fetches.find(x => x.responded && x.sourceConfig.url === source.config.url);
230
+ if (fetch) {
231
+ source.seedLatency(fetch.latency);
232
+ }
233
+ }
225
234
  this.activeConfig = active;
226
235
  this.startConfigPoll();
227
236
  return { config: active, sources };
@@ -295,7 +304,13 @@ export class ArchivesChain implements IArchives {
295
304
 
296
305
  private async adoptNewConfig(state: ChainState, latest: RemoteConfig): Promise<void> {
297
306
  if (JSON.stringify(latest) === JSON.stringify(state.config)) return;
298
- console.log(`Storage routing config changed for ${this.getDebugName()}, rebuilding sources`);
307
+ // Same version but different content is a deploy switchover remap (an in-memory window
308
+ // split pointing at a successor's port), not an actual configuration change
309
+ if (getConfigVersion(latest) === getConfigVersion(state.config)) {
310
+ console.log(`Storage routing reinterpreted at version ${getConfigVersion(latest)} for ${this.getDebugName()} (deploy switchover in progress), rebuilding sources. Sources: ${JSON.stringify(latest.sources)}`);
311
+ } else {
312
+ console.log(`Storage routing config changed for ${this.getDebugName()} (version ${getConfigVersion(state.config)} -> ${getConfigVersion(latest)}), rebuilding sources`);
313
+ }
299
314
  let oldByConfig = new Map(state.sources.map(source => [JSON.stringify(source.config), source]));
300
315
  let sources: SourceWrapper[] = [];
301
316
  for (let sourceConfig of latest.sources.map(normalizeSource)) {
@@ -342,6 +357,7 @@ export class ArchivesChain implements IArchives {
342
357
  // Init is failing; its own retry loop handles that
343
358
  return;
344
359
  }
360
+ console.log(`Every storage source failed for ${this.getDebugName()}; re-contacting all ${state.sources.length} sources (routing re-read + connection re-attempt)`);
345
361
  let results = await Promise.all(state.sources.map(async source => {
346
362
  try {
347
363
  let data = await source.read(archives => archives.get(ROUTING_FILE));
@@ -456,12 +472,14 @@ export class ArchivesChain implements IArchives {
456
472
  let now = Date.now();
457
473
  let nearBoundary = state.sources.some(source => source.config.validWindow.some(t => t > 0 && t < Number.MAX_SAFE_INTEGER && Math.abs(t - now) <= WRONG_TARGET_BOUNDARY_WINDOW));
458
474
  if (nearBoundary) {
475
+ console.log(`Write rejected by ${this.getDebugName()}: raced a valid window boundary; waiting ${WRONG_TARGET_BOUNDARY_RETRY_DELAY / 1000}s and retrying`);
459
476
  await delay(WRONG_TARGET_BOUNDARY_RETRY_DELAY);
460
477
  return;
461
478
  }
462
479
  }
463
480
  if (Date.now() - this.lastConfigRefresh < CONFIG_REFRESH_THROTTLE) return;
464
481
  this.lastConfigRefresh = Date.now();
482
+ console.log(`Write rejected by ${this.getDebugName()} (wrong ${kind}): our config is stale (likely a deploy switchover); refreshing it and retrying`);
465
483
  await this.refreshActiveConfig();
466
484
  }
467
485
 
@@ -665,6 +683,7 @@ export class ArchivesChain implements IArchives {
665
683
  // NOTE: If we run into cases when transport level errors happen, but we are still connected, then we might want to add additional checking here, additional errors in which we will retry the other targets. However, ideally, checking if the WebSocket connection is still connected will handle all those cases. Distinguishing between application errors (which we can't retry) and transport errors.
666
684
  if (target.isConnected()) throw e;
667
685
  target.noteFailure();
686
+ console.log(`Variable-shard write target ${target.getDebugName()} is down; moving to the next-lowest-latency shard`);
668
687
  errors.push(String((e as Error).stack ?? e));
669
688
  }
670
689
  }
@@ -20,7 +20,6 @@ import { parseHostedUrl, replaceHostedUrlPort } from "./remoteConfig";
20
20
  // any point during a switchover.
21
21
 
22
22
  const PARAMETERS_TIMELINE_FILE_REGEX = /^(\d+)-parameters\.json$/;
23
- const ALIVE_WINDOW_FOREVER = Number.MAX_SAFE_INTEGER;
24
23
  const TIMELINE_POLL_INTERVAL = 60 * 1000;
25
24
  // Processes listening on an alternate port register it here (sibling of the storage folder), keyed
26
25
  // by their actual pid; content includes their ancestor pids so the timeline entry pid (a shell
@@ -42,8 +41,10 @@ const REMAP_EXPIRE_FRACTION = 2;
42
41
 
43
42
  type TimelineEntry = {
44
43
  pid?: number;
45
- aliveWindow: [number, number];
46
- parameters: unknown;
44
+ // releaseTime + overlapTime define everything: the newer entry's releaseTime is when it
45
+ // starts, and releaseTime + overlapTime is when the older one is killed (freeing the port).
46
+ // The files also carry an aliveWindow, which is deliberately ignored.
47
+ parameters: { releaseTime?: number; overlapTime?: number };
47
48
  };
48
49
 
49
50
  type TakeoverComputed = {
@@ -67,6 +68,8 @@ let current: TakeoverComputed = {};
67
68
  let currentKey = JSON.stringify(current);
68
69
  let eventListeners: ((event: TakeoverEvent) => void)[] = [];
69
70
  let eventTimers: ReturnType<typeof setTimeout>[] = [];
71
+ let loggedNoMatch = false;
72
+ let loggedIdentity = false;
70
73
 
71
74
  function execText(command: string): Promise<string> {
72
75
  return new Promise((resolve, reject) => {
@@ -103,20 +106,10 @@ async function getAncestorPids(): Promise<Set<number>> {
103
106
  return pids;
104
107
  }
105
108
 
106
- async function findTimelineFolder(): Promise<string | undefined> {
107
- let dir = process.cwd();
108
- while (true) {
109
- let files: string[] = [];
110
- try {
111
- files = await fs.promises.readdir(dir);
112
- } catch { }
113
- if (files.some(file => PARAMETERS_TIMELINE_FILE_REGEX.test(file))) {
114
- return dir.replaceAll("\\", "/") + "/";
115
- }
116
- let parent = path.dirname(dir);
117
- if (parent === dir) return undefined;
118
- dir = parent;
119
- }
109
+ // The instance folder is ALWAYS the parent of our cwd (services run in <instance>/git/) - never
110
+ // search for it, a walk could latch onto the wrong folder
111
+ function getTimelineFolder(): string {
112
+ return path.dirname(process.cwd()).replaceAll("\\", "/") + "/";
120
113
  }
121
114
 
122
115
  function pidAlive(pid: number): boolean {
@@ -211,8 +204,11 @@ function scheduleEvents(): void {
211
204
  };
212
205
  let remap = current.remap;
213
206
  if (remap) {
207
+ let altPort = remap.altPort;
214
208
  schedule(remap.boundaryA - SWITCH_SCAN_LEAD, () => emit("diskScan"));
209
+ schedule(remap.boundaryA, () => console.log(`Deploy switchover write handoff boundary passed: fresh writes now belong to the alternate-port side (port ${altPort})`));
215
210
  schedule(remap.boundaryA + SWITCH_SCAN_LEAD, () => emit("diskScan"));
211
+ schedule(remap.boundaryB, () => console.log(`Deploy switchover second boundary passed: fresh writes return to the main port`));
216
212
  schedule(remap.expire, () => void recompute().catch((e: Error) => console.error(`Deploy takeover recompute failed: ${e.stack ?? e}`)));
217
213
  }
218
214
  }
@@ -228,26 +224,42 @@ async function recompute(): Promise<void> {
228
224
  us = entry;
229
225
  }
230
226
  }
227
+ // Matching failures must be loud, or takeover detection fails silently forever
228
+ if (!us && entries.length && !loggedNoMatch) {
229
+ loggedNoMatch = true;
230
+ console.warn(`No deploy timeline entry matches our pid chain (${[...ancestorPids].join(", ")}) - takeover detection cannot work. Entries: ${entries.map(x => `pid=${x.pid ?? "none"} releaseTime=${x.parameters.releaseTime}`).join(" | ")}`);
231
+ }
232
+ if (us && !loggedIdentity) {
233
+ loggedIdentity = true;
234
+ console.log(`Deploy timeline entry matched: we are pid=${us.pid}, releaseTime=${us.parameters.releaseTime !== undefined && iso(us.parameters.releaseTime) || "unknown"}, ${entries.length} entries total`);
235
+ }
231
236
  let computed: TakeoverComputed = {};
232
- if (us) {
233
- let usStart = us.aliveWindow[0];
234
- let next: TimelineEntry | undefined;
235
- let prev: TimelineEntry | undefined;
237
+ let usRelease = us?.parameters.releaseTime;
238
+ if (us && usRelease !== undefined) {
239
+ // The newer entry's releaseTime is when it starts; releaseTime + overlapTime is when the
240
+ // older one is killed. That's the whole timeline.
241
+ let next: { entry: TimelineEntry; release: number } | undefined;
242
+ let prev: { entry: TimelineEntry; release: number } | undefined;
236
243
  for (let entry of entries) {
237
244
  if (entry === us) continue;
238
- if (entry.aliveWindow[0] > usStart && (!next || entry.aliveWindow[0] < next.aliveWindow[0])) {
239
- next = entry;
245
+ let release = entry.parameters.releaseTime;
246
+ if (release === undefined) continue;
247
+ if (release > usRelease && (!next || release < next.release)) {
248
+ next = { entry, release };
240
249
  }
241
- if (entry.aliveWindow[0] < usStart && (!prev || entry.aliveWindow[0] > prev.aliveWindow[0])) {
242
- prev = entry;
250
+ if (release < usRelease && (!prev || release > prev.release)) {
251
+ prev = { entry, release };
243
252
  }
244
253
  }
245
- if (next && us.aliveWindow[1] !== ALIVE_WINDOW_FOREVER && next.aliveWindow[0] <= us.aliveWindow[1]) {
246
- let overlap = us.aliveWindow[1] - next.aliveWindow[0];
247
- let successorStart = next.aliveWindow[0];
254
+ if (next) {
255
+ let successorStart = next.release;
256
+ let overlap = next.entry.parameters.overlapTime || us.parameters.overlapTime || 0;
257
+ // Even a ZERO-overlap deploy needs the dying behaviors - draining fast-write flushes
258
+ // by the release and writing through after it, or acknowledged data dies with us.
259
+ // Only the remap (the alternate-port middle window) needs an actual overlap.
260
+ computed.dying = { successorStart, overlap };
248
261
  if (overlap > 0 && now < successorStart + overlap * REMAP_EXPIRE_FRACTION) {
249
- computed.dying = { successorStart, overlap };
250
- const successorPid = next.pid;
262
+ const successorPid = next.entry.pid;
251
263
  if (successorPid) {
252
264
  let successorEntry = registry.find(x => x.ancestorPids.includes(successorPid));
253
265
  if (successorEntry) {
@@ -256,31 +268,58 @@ async function recompute(): Promise<void> {
256
268
  }
257
269
  }
258
270
  }
259
- if (prev && prev.aliveWindow[1] > usStart) {
260
- let overlap = prev.aliveWindow[1] - usStart;
261
- computed.predecessorEnd = prev.aliveWindow[1];
262
- if (overlap > 0 && ourAltPort && now < usStart + overlap * REMAP_EXPIRE_FRACTION && !computed.remap) {
263
- computed.remap = makeRemap(usStart, overlap, ourAltPort);
271
+ if (prev) {
272
+ let overlap = us.parameters.overlapTime || prev.entry.parameters.overlapTime || 0;
273
+ // The predecessor holds the main port until OUR release + the overlap
274
+ computed.predecessorEnd = usRelease + overlap;
275
+ if (overlap > 0 && ourAltPort && now < usRelease + overlap * REMAP_EXPIRE_FRACTION && !computed.remap) {
276
+ computed.remap = makeRemap(usRelease, overlap, ourAltPort);
264
277
  }
265
278
  }
266
279
  }
267
280
  let key = JSON.stringify(computed);
268
281
  if (key === currentKey) return;
282
+ logTransitions(current, computed);
269
283
  current = computed;
270
284
  currentKey = key;
271
- console.log(`Deploy takeover state changed: ${key}`);
272
285
  scheduleEvents();
273
286
  emit("remapChanged");
274
287
  }
275
288
 
289
+ function iso(time: number): string {
290
+ return new Date(time).toISOString();
291
+ }
292
+
293
+ // The switchover lifecycle is rare and important, so every stage gets a clear line
294
+ function logTransitions(prev: TakeoverComputed, next: TakeoverComputed): void {
295
+ if (!prev.dying && next.dying) {
296
+ let { successorStart, overlap } = next.dying;
297
+ console.log(`Deploy switchover scheduled: our successor starts at ${iso(successorStart)}; fast-write flushes drain by ${iso(successorStart + overlap * FLUSH_DEADLINE_FRACTION)}, fresh writes hand off at ${iso(successorStart + overlap * BOUNDARY_A_FRACTION)}, and we are killed at ${iso(successorStart + overlap)}`);
298
+ }
299
+ if (!prev.predecessorEnd && next.predecessorEnd) {
300
+ console.log(`We are a deploy successor: our predecessor holds the main port until ${iso(next.predecessorEnd)}`);
301
+ }
302
+ if (!prev.remap && next.remap) {
303
+ console.log(`Deploy switchover remap active: sources pointing at us are split so [${iso(next.remap.boundaryA)} .. ${iso(next.remap.boundaryB)}] routes to alternate port ${next.remap.altPort} (remap expires at ${iso(next.remap.expire)})`);
304
+ }
305
+ if (prev.remap && !next.remap) {
306
+ console.log(`Deploy switchover remap ended; running on the plain stored routing config`);
307
+ }
308
+ }
309
+
276
310
  /** Starts the takeover machinery. Port fallback (alternate port + registry + acquisition polling)
277
311
  * works regardless; without a deploy timeline folder the switchover-specific parts (the remap,
278
312
  * the flush deadline, the tighter acquisition pacing) simply stay inert. */
279
313
  export async function initDeployTakeover(config: { domain: string; mainPort: number; storageFolder: string }): Promise<void> {
280
314
  initialized = config;
281
315
  ancestorPids = await getAncestorPids();
282
- timelineFolder = await findTimelineFolder();
283
- if (!timelineFolder) return;
316
+ timelineFolder = getTimelineFolder();
317
+ let entries = await readTimelineEntries();
318
+ if (!entries.length) {
319
+ console.log(`No deploy timeline entries in ${timelineFolder}; deploy takeover inactive until they appear (port fallback still works)`);
320
+ } else {
321
+ console.log(`Deploy timeline found at ${timelineFolder} (${entries.length} entries); our pid chain: ${[...ancestorPids].join(" -> ")}`);
322
+ }
284
323
  await recompute();
285
324
  let poll = setInterval(() => {
286
325
  void recompute().catch((e: Error) => console.error(`Deploy takeover recompute failed: ${e.stack ?? e}`));