querysub 0.538.0 → 0.540.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.538.0",
+  "version": "0.540.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -71,7 +71,7 @@
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
-    "sliftutils": "^1.7.21",
+    "sliftutils": "^1.7.23",
     "socket-function": "^1.2.26",
     "terser": "^5.31.0",
     "typenode": "^6.6.1",

package/src/-a-archives/archives2.ts

@@ -0,0 +1,109 @@
+import { IArchives, RemoteConfigBase } from "sliftutils/storage/IArchives";
+import { createArchives } from "sliftutils/storage/remoteStorage/createArchives";
+import { cache, cacheJSONArgsEqual } from "socket-function/src/caching";
+import { formatDateTime } from "socket-function/src/formatting/format";
+import { getDomain } from "../config";
+
+function createSourceWindows(
+    overrides: Partial<RemoteConfigBase>,
+    sourceWindows: {
+        startTime: number;
+        sources: Omit<Exclude<RemoteConfigBase, string>, "validWindow">[];
+    }[]
+) {
+    let sources: RemoteConfigBase[] = [];
+
+    let timeRanges: [number, number][] = [];
+    for (let windowIndex = 0; windowIndex < sourceWindows.length; windowIndex++) {
+        let startTime = sourceWindows[windowIndex].startTime;
+        let endTime = sourceWindows[windowIndex + 1]?.startTime ?? Number.MAX_SAFE_INTEGER;
+        if (startTime > endTime) throw new Error(`Source window ${formatDateTime(startTime)} starts after end of ${formatDateTime(endTime)}`);
+        timeRanges.push([startTime, endTime]);
+    }
+
+    // NOTE: We scan from everything after us, so I don't believe we need any interlacing here
+    for (let windowIndex = 0; windowIndex < sourceWindows.length; windowIndex++) {
+        for (let win of sourceWindows[windowIndex].sources) {
+            let fullWindow = { ...win, ...overrides as {}, validWindow: timeRanges[windowIndex] };
+            sources.push(fullWindow);
+        }
+    }
+    return sources;
+}
+
+export function archiveBuilder(bucket: string, overrides: Partial<RemoteConfigBase>) {
+    const HETZNER = `https://65-109-93-113.querysubtest.com:5233/file/root/${bucket}/storage/storagerouting.json`;
+    const ONTARIO = `https://99-250-124-91.querysubtest.com:5234/file/root/${bucket}/storage/storagerouting.json`;
+    const BACKBLAZE = `https://f002.backblazeb2.com/file/querysub-storagetest/${bucket}/storage/storagerouting.json`;
+    let sources = createSourceWindows(overrides, [
+        {
+            startTime: 0,
+            sources: [
+                { type: "remote", url: HETZNER, route: [0.5, 1], },
+                { type: "remote", url: ONTARIO, route: [0, 0.5], },
+                { type: "remote", url: HETZNER, },
+                { type: "remote", url: ONTARIO, },
+                { type: "backblaze", url: BACKBLAZE, },
+            ]
+        },
+        // {
+        //     startTime: +new Date("2026-07-20 12:00:00-04:00"),
+        //     sources: [
+        //         { type: "remote", url: HETZNER, route: [0.5, 1], },
+        //         { type: "remote", url: ONTARIO, route: [0, 0.5], },
+        //         { type: "remote", url: HETZNER, },
+        //         { type: "remote", url: ONTARIO, },
+        //         { type: "backblaze", url: BACKBLAZE, },
+        //     ]
+        // }
+    ]);
+
+    return createArchives({
+        version: 7,
+        sources,
+    });
+}
+
+// TODO: We'll turn on full sync eventually. That way we can be backblaze agnostic. For now we'll leave full sync off so we don't have to download hundreds of gigabytes and potentially change our mind about how we want to store it. 
+
+const archivesBuilderCache = cacheJSONArgsEqual((domain: string, overrides: Partial<RemoteConfigBase>) => {
+    return archiveBuilder(domain, overrides);
+});
+function nestBucket(bucket: string, archives: IArchives): IArchives {
+    if (!bucket) return archives;
+    if (!bucket.endsWith("/")) {
+        bucket = bucket + "/";
+    }
+    return {
+        get: (path: string) => archives.get(bucket + path),
+        set: (path: string, data: Buffer) => archives.set(bucket + path, data),
+        del: (path: string) => archives.del(bucket + path),
+        find: (prefix: string) => archives.find(bucket + prefix),
+        findInfo: (prefix: string) => archives.findInfo(bucket + prefix),
+        get2: (path: string) => archives.get2(bucket + path),
+        setLargeFile: (config: { path: string; getNextData(): Promise<Buffer | undefined>; }) => archives.setLargeFile({ path: bucket + config.path, getNextData: config.getNextData }),
+        getInfo: (path: string) => archives.getInfo(bucket + path),
+        getDebugName: () => archives.getDebugName() + "/" + bucket,
+        hasWriteAccess: () => archives.hasWriteAccess(),
+        getConfig: () => archives.getConfig(),
+        getURL: (path: string) => archives.getURL(bucket + path),
+    };
+}
+
+export function getArchives2(bucket: string) {
+    return nestBucket(bucket, archivesBuilderCache(getDomain(), { noFullSync: true, }));
+}
+export function getArchives2PrivateImmutable(bucket: string) {
+    return nestBucket(bucket, archivesBuilderCache(getDomain() + "-private-immutable", { noFullSync: true, immutable: true }));
+}
+function getAllowedOrigins(domain: string) {
+    return [`https://${domain}`, `https://127-0-0-1.${domain}:7007`];
+}
+export function getArchives2PublicImmutable(bucket: string) {
+    let domain = getDomain();
+    return nestBucket(bucket, archivesBuilderCache(domain + "-public-immutable", { noFullSync: true, immutable: true, public: true, allowedOrigins: getAllowedOrigins(domain) }));
+}
+export function getArchives2Public(bucket: string) {
+    let domain = getDomain();
+    return nestBucket(bucket, archivesBuilderCache(domain + "-public", { noFullSync: true, public: true, allowedOrigins: getAllowedOrigins(domain) }));
+}

package/src/deployManager/machineApplyMainCode.ts

@@ -455,6 +455,8 @@ const runScreenCommand = measureWrap(async function runScreenCommand(config: {
     command: string;
     // Defaults to the folder derived from screenName; a future screen passes its canonical screen's folder
     folder?: string;
+    // Defaults to "pipe"; a future screen uses its own pipe file so its output never interleaves with the old screen's
+    pipeName?: string;
 }) {
     let prefix = getTmuxPrefix();
     let screenName = config.screenName;
@@ -491,6 +493,7 @@ const runScreenCommand = measureWrap(async function runScreenCommand(config: {
                     screenName,
                     command: config.command,
                     folder: config.folder,
+                    pipeName: config.pipeName,
                 });
                 return;
             }
@@ -505,10 +508,14 @@ ${config.command}
     await fs.promises.writeFile(folder + "command.sh", command);
     await runPromise(`${prefix}tmux send-keys -t ${screenName} 'bash ../command.sh' Enter`);
 
-    // Setup pipe-pane as well
+    await setupPipePane({ screenName, folder, pipeName: config.pipeName || "pipe" });
+});
 
-    let pipeFile = path.resolve(folder + "pipe.txt");
-    let pipeScript = path.resolve(folder + "pipe.sh");
+// Writes the rolling-buffer pipe script and (re)attaches the screen's pipe-pane to it. pipeName distinguishes the future screen's log from the canonical one in the shared folder — two screens' outputs must never interleave into one file.
+async function setupPipePane(config: { screenName: string; folder: string; pipeName: string }) {
+    let prefix = getTmuxPrefix();
+    let pipeFile = path.resolve(config.folder + config.pipeName + ".txt");
+    let pipeScript = path.resolve(config.folder + config.pipeName + ".sh");
     await fs.promises.writeFile(pipeScript, `#!/bin/bash
 line_count=0
 while IFS= read -r line; do
@@ -532,8 +539,8 @@ while IFS= read -r line; do
     fi
 done`);
     await runPromise(`chmod +x ${pipeScript}`);
-    await runPromise(`${prefix}tmux pipe-pane -t ${screenName} 'bash ${pipeScript}'`);
-});
+    await runPromise(`${prefix}tmux pipe-pane -t ${config.screenName} 'bash ${pipeScript}'`);
+}
 const killScreen = measureWrap(async function killScreen(config: {
     screenName: string;
     // During a takeover the folder's nodeId file already belongs to the NEW process, so the old screen's kill must not remove it
@@ -704,7 +711,7 @@ async function ensureFutureStarted(config: {
         takeoverOldNodeIds.set(canonicalScreenName, await fs.promises.readFile(nodeIdFile, "utf8"));
     }
     console.log(green(`Starting future instance ${magenta(futureScreenName)} (release time reached)`));
-    await runScreenCommand({ screenName: futureScreenName, folder, command: next.command });
+    await runScreenCommand({ screenName: futureScreenName, folder, command: next.command, pipeName: "pipe-future" });
 }
 
 // Kills the old canonical screen (if any) and renames the future screen to the canonical name. The rename leaves the new process completely untouched, so the takeover is seamless. No-op when there is no future screen.
@@ -733,6 +740,15 @@ async function takeoverFutureScreen(config: {
     await runPromise(`${prefix}tmux rename-session -t ${futureScreenName} ${canonicalScreenName}`);
     screenStateMap.set(canonicalScreenName, { ...future, screenName: canonicalScreenName });
     screenStateMap.delete(futureScreenName);
+    // The screen changed, so its log is replaced wholesale: the future screen's output becomes the canonical log (never interleaved with the old process's output), and its pipe-pane is re-attached to the canonical pipe file for everything after the rename
+    let folder = os.homedir() + "/" + SERVICE_FOLDER + canonicalScreenName + "/";
+    try {
+        await fs.promises.rename(folder + "pipe-future.txt", folder + "pipe.txt");
+    } catch {
+        // The future screen produced no output yet; still clear the old screen's log
+        await fs.promises.writeFile(folder + "pipe.txt", "");
+    }
+    await setupPipePane({ screenName: canonicalScreenName, folder, pipeName: "pipe" });
 }
 
 function sameRestartParameters(prevParametersJSON: string, next: ServiceParameters): boolean {