querysub 0.442.0 → 0.446.0

package/src/deployManager/machineApplyMainCode.ts

@@ -26,7 +26,12 @@ import { PromiseObj } from "../promise";
 import path from "path";
 import { fsExistsAsync } from "../fs";
 
-const PIPE_FILE_LINE_LIMIT = 10_000;
+// The deployed-service stdout log (pipe.txt) is a rolling buffer: once it grows
+// to PIPE_FILE_LINE_LIMIT lines it is truncated down to PIPE_FILE_LINE_KEEP. The
+// file size is checked every PIPE_FILE_LINE_KEEP lines, so the file stays within
+// [PIPE_FILE_LINE_KEEP, PIPE_FILE_LINE_LIMIT] lines.
+const PIPE_FILE_LINE_LIMIT = 2_000;
+const PIPE_FILE_LINE_KEEP = 1_000;
 
 
 const getMemoryInfo = measureWrap(async function getMemoryInfo(): Promise<{ value: number; max: number } | undefined> {
@@ -551,16 +556,16 @@ while IFS= read -r line; do
     echo "$line" >> "${pipeFile}"
     ((line_count++))
     
-    # Check line count every ${PIPE_FILE_LINE_LIMIT} lines to avoid too much overhead
-    if (( line_count % ${PIPE_FILE_LINE_LIMIT} == 0 )); then
+    # Check line count every ${PIPE_FILE_LINE_KEEP} lines to avoid too much overhead
+    if (( line_count % ${PIPE_FILE_LINE_KEEP} == 0 )); then
         if [ -f "${pipeFile}" ]; then
             # Count total lines in file
             total_lines=$(wc -l < "${pipeFile}")
-            if [ "$total_lines" -gt ${PIPE_FILE_LINE_LIMIT} ]; then
+            if [ "$total_lines" -ge ${PIPE_FILE_LINE_LIMIT} ]; then
                 # Wait 2 seconds to give the watcher time to read pending content
                 sleep 2
-                # Keep only the last ${PIPE_FILE_LINE_LIMIT} lines when file gets too many lines
-                tail -n ${Math.floor(PIPE_FILE_LINE_LIMIT / 2)} "${pipeFile}" > "${pipeFile}.tmp" && mv "${pipeFile}.tmp" "${pipeFile}"
+                # Keep only the last ${PIPE_FILE_LINE_KEEP} lines when file gets too many lines
+                tail -n ${PIPE_FILE_LINE_KEEP} "${pipeFile}" > "${pipeFile}.tmp" && mv "${pipeFile}.tmp" "${pipeFile}"
                 # AND, give it time to read the truncation
                 sleep 2
             fi

package/src/deployManager/machineSchema.ts

@@ -13,7 +13,7 @@ import { getOwnMachineId } from "../-a-auth/certs";
 import { delay, runInSerial, runInfinitePollCallAtStart } from "socket-function/src/batching";
 import { lazy } from "socket-function/src/caching";
 import { errorToUndefinedSilent } from "../errors";
-import { commitAndPush, getGitRefInfo, getGitRefLive, getGitURLLive, getGitUncommitted, getLatestRefOnUpstreamBranch, setGitRef } from "../4-deploy/git";
+import { commitAndPush, getGitDiff, getGitRefInfo, getGitRefLive, getGitURLLive, getGitUncommitted, getLatestRefOnUpstreamBranch, setGitRef } from "../4-deploy/git";
 import { OnServiceChange } from "./machineController";
 import path from "path";
 import fs from "fs";
@@ -291,6 +291,21 @@ export class MachineServiceControllerBase {
         }
     }
 
+    /** Returns a unified-diff-style summary of all uncommitted changes, for the
+     *      same repo `commitPushService` / `commitPushAndPublishQuerysub` would commit. */
+    public async getGitDiff(config: {
+        useQuerysub?: boolean;
+    }): Promise<string> {
+        let gitDir = ".";
+        if (config.useQuerysub) {
+            gitDir = path.resolve("../querysub");
+            if (!await fsExistsAsync(gitDir)) {
+                throw new Error(`Querysub folder does not exist at ${gitDir}`);
+            }
+        }
+        return await getGitDiff(gitDir);
+    }
+
     public async commitPushService(commitMessage: string) {
         if (commitMessage.toLowerCase().includes("querysub")) {
             let querysubFolder = path.resolve("../querysub");
@@ -448,6 +463,7 @@ export const MachineServiceController = getSyncedController(
             deleteServiceConfig: {},
             getGitInfo: {},
             getGitRefInfo: {},
+            getGitDiff: {},
             commitPushService: {},
             commitPushAndPublishQuerysub: {},
             getPendingFunctions: {},

package/src/diagnostics/debugger/debugger-remote.ts

@@ -0,0 +1,231 @@
+/**
+ * DebuggerRemote — a small wrapper around a remote Node.js V8 inspector.
+ *
+ * Connects over the Chrome DevTools Protocol (CDP) and exposes `evaluate()`
+ * for running expressions inside the remote process. Connecting is lazy: the
+ * constructor kicks it off, and every request awaits it, so callers can just
+ * `new DebuggerRemote(...)` and `await remote.evaluate(...)`.
+ *
+ * Copied from https://github.com/sliftist/debug
+ */
+import * as fs from "fs";
+
+/** How the remote inspector should be located. */
+export type DebuggerRemoteOptions =
+    /** A ready-to-use CDP WebSocket URL (ws://host:port/<uuid>). */
+    | { wsUrl: string }
+    /** An inspector HTTP port; the live WebSocket URL is looked up. */
+    | { port: number; host?: string }
+    /** The `debugger-port.json` file written by the target script. */
+    | { portFile: string };
+
+interface InspectorTarget {
+    webSocketDebuggerUrl?: string;
+}
+
+interface PortFile {
+    port: number;
+    url: string;
+    pid: number;
+}
+
+interface PendingRequest {
+    resolve: (result: any) => void;
+    reject: (err: Error) => void;
+}
+
+const DEFAULT_TIMEOUT_MS = 10_000;
+const CONNECT_MAX_ATTEMPTS = 10;
+const CONNECT_RETRY_DELAY_MS = 1_000;
+
+function delay(ms: number): Promise<void> {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+export class DebuggerRemote {
+    private ready: Promise<WebSocket>;
+    private nextId = 1;
+    private pending = new Map<number, PendingRequest>();
+    private eventListeners = new Map<string, Set<(params: any) => void>>();
+    private closed = false;
+
+    constructor(private options: DebuggerRemoteOptions) {
+        this.ready = this.connect();
+        // Avoid an unhandled-rejection warning if nobody awaits a request
+        // before the connection itself fails; real callers still see the error.
+        this.ready.catch(() => {});
+    }
+
+    /** Evaluate a JS expression inside the remote process and return its value. */
+    async evaluate(expression: string): Promise<unknown> {
+        const response = await this.send("Runtime.evaluate", {
+            expression,
+            returnByValue: true,
+            awaitPromise: true,
+        });
+        if (response.exceptionDetails) {
+            throw new Error(`Remote threw: ${JSON.stringify(response.exceptionDetails)}`);
+        }
+        return response.result.value;
+    }
+
+    /**
+     * Subscribe to a CDP event notification (e.g. "Debugger.scriptParsed",
+     * "Runtime.consoleAPICalled"). Returns an unsubscribe function.
+     */
+    on(method: string, listener: (params: any) => void): () => void {
+        let listeners = this.eventListeners.get(method);
+        if (!listeners) {
+            listeners = new Set();
+            this.eventListeners.set(method, listeners);
+        }
+        listeners.add(listener);
+        return () => {
+            listeners!.delete(listener);
+        };
+    }
+
+    /** Send a raw CDP command and await its result. */
+    send(method: string, params: object = {}, timeoutMs = DEFAULT_TIMEOUT_MS): Promise<any> {
+        return this.ready.then(
+            (ws) =>
+                new Promise<any>((resolve, reject) => {
+                    if (this.closed) {
+                        reject(new Error("DebuggerRemote is closed"));
+                        return;
+                    }
+                    const id = this.nextId++;
+                    const timeout = setTimeout(() => {
+                        this.pending.delete(id);
+                        reject(new Error(`Timed out after ${timeoutMs}ms waiting for "${method}"`));
+                    }, timeoutMs);
+                    this.pending.set(id, {
+                        resolve: (result) => {
+                            clearTimeout(timeout);
+                            resolve(result);
+                        },
+                        reject: (err) => {
+                            clearTimeout(timeout);
+                            reject(err);
+                        },
+                    });
+                    ws.send(JSON.stringify({ id, method, params }));
+                }),
+        );
+    }
+
+    /** Close the connection and reject any in-flight requests. */
+    close(): void {
+        this.closed = true;
+        this.ready.then((ws) => ws.close(), () => {});
+    }
+
+    /**
+     * Connect, retrying on any error so `attach` can be started before the
+     * target is up (missing port file, inspector not listening, etc.).
+     */
+    private async connect(): Promise<WebSocket> {
+        let lastErr: unknown;
+        for (let attempt = 1; attempt <= CONNECT_MAX_ATTEMPTS; attempt++) {
+            if (this.closed) throw new Error("DebuggerRemote was closed before connecting");
+            try {
+                return await this.tryConnect();
+            } catch (err) {
+                lastErr = err;
+                const message = (err as Error)?.message ?? String(err);
+                if (attempt < CONNECT_MAX_ATTEMPTS) {
+                    // stderr, not stdout — callers may use stdout for protocol I/O.
+                    console.error(
+                        `[DebuggerRemote] connect attempt ${attempt}/${CONNECT_MAX_ATTEMPTS} ` +
+                            `failed (${message}); retrying in ${CONNECT_RETRY_DELAY_MS}ms...`,
+                    );
+                    await delay(CONNECT_RETRY_DELAY_MS);
+                }
+            }
+        }
+        throw new Error(
+            `Could not connect to inspector after ${CONNECT_MAX_ATTEMPTS} attempts: ` +
+                `${(lastErr as Error)?.message ?? lastErr}`,
+        );
+    }
+
+    /** A single connection attempt: resolve the URL and open the WebSocket. */
+    private async tryConnect(): Promise<WebSocket> {
+        const wsUrl = await this.resolveWsUrl();
+        const ws = new WebSocket(wsUrl);
+        await new Promise<void>((resolve, reject) => {
+            ws.addEventListener("open", () => resolve(), { once: true });
+            ws.addEventListener(
+                "error",
+                () => reject(new Error(`Could not connect to inspector at ${wsUrl}`)),
+                { once: true },
+            );
+        });
+        ws.addEventListener("message", (event) => this.onMessage(event));
+        ws.addEventListener("close", () => this.onClose());
+        return ws;
+    }
+
+    /** Turn the constructor options into a concrete CDP WebSocket URL. */
+    private async resolveWsUrl(): Promise<string> {
+        const opts = this.options;
+        if ("wsUrl" in opts) {
+            return opts.wsUrl;
+        }
+
+        let port: number;
+        let host = "127.0.0.1";
+        if ("portFile" in opts) {
+            if (!fs.existsSync(opts.portFile)) {
+                throw new Error(
+                    `Port file not found: ${opts.portFile}\n` +
+                        `Start the target first with \`yarn test\`.`,
+                );
+            }
+            const info = JSON.parse(fs.readFileSync(opts.portFile, "utf8")) as PortFile;
+            port = info.port;
+        } else {
+            port = opts.port;
+            if (opts.host) host = opts.host;
+        }
+
+        // The UUID in the WebSocket URL changes per process, so always look it
+        // up fresh from the inspector's HTTP endpoint.
+        const res = await fetch(`http://${host}:${port}/json/list`);
+        const targets = (await res.json()) as InspectorTarget[];
+        const wsUrl = targets[0]?.webSocketDebuggerUrl;
+        if (!wsUrl) {
+            throw new Error(`No webSocketDebuggerUrl returned by inspector on port ${port}`);
+        }
+        return wsUrl;
+    }
+
+    private onMessage(event: MessageEvent): void {
+        const msg = JSON.parse(event.data as string);
+        if (typeof msg.id !== "number") {
+            // A CDP event notification: { method, params }, no `id`.
+            if (typeof msg.method === "string") {
+                const listeners = this.eventListeners.get(msg.method);
+                if (listeners) {
+                    for (const listener of [...listeners]) listener(msg.params ?? {});
+                }
+            }
+            return;
+        }
+        const pending = this.pending.get(msg.id);
+        if (!pending) return;
+        this.pending.delete(msg.id);
+        if (msg.error) {
+            pending.reject(new Error(`CDP error: ${JSON.stringify(msg.error)}`));
+        } else {
+            pending.resolve(msg.result);
+        }
+    }
+
+    private onClose(): void {
+        for (const pending of this.pending.values()) {
+            pending.reject(new Error("Inspector connection closed"));
+        }
+        this.pending.clear();
+    }
+}