querysub 0.525.0 → 0.527.0

package/.claude/settings.local.json

@@ -29,7 +29,8 @@
       "WebSearch",
       "WebFetch(domain:blog.cloudflare.com)",
       "WebFetch(domain:developers.cloudflare.com)",
-      "PowerShell(yarn type *)"
+      "PowerShell(yarn type *)",
+      "PowerShell(Remove-Item -Confirm:$false \"C:\\\\Users\\\\quent\\\\.claude\\\\projects\\\\D--repos-querysub\\\\memory\\\\no-naming-new-bins.md\")"
     ]
   }
 }

package/appSecrets.ts

@@ -0,0 +1,23 @@
+import fs from "fs";
+import { getBackblazePath } from "./src/-a-archives/archivesBackBlaze";
+import { getCloudflareCreds } from "./src/-b-authorities/cloudflareHelpers";
+import { Querysub } from "./src/4-querysub/Querysub";
+
+/** Serves the secret keys sliftutils' getSecret expects, reading them the way querysub already does: backblaze creds off disk (getBackblazePath), cloudflare creds from the keys archives bucket (with its local file fallback). */
+export async function getAppSecret(key: string): Promise<string | undefined> {
+    // Use this so some of our default imports get imported
+    Querysub;
+    if (key.startsWith("backblaze.json.")) {
+        let creds = JSON.parse(fs.readFileSync(getBackblazePath(), "utf8")) as { applicationKeyId?: string; applicationKey?: string };
+        if (key === "backblaze.json.applicationKeyId") return creds.applicationKeyId;
+        if (key === "backblaze.json.applicationKey") return creds.applicationKey;
+        return undefined;
+    }
+    if (key.startsWith("cloudflare.json.")) {
+        let creds = await getCloudflareCreds();
+        if (key === "cloudflare.json.key") return creds.key;
+        if (key === "cloudflare.json.email") return creds.email;
+        return undefined;
+    }
+    return undefined;
+}

package/bin/storageserve.js

@@ -0,0 +1,8 @@
+#!/usr/bin/env node
+
+require("typenode");
+require("../src/inject.ts");
+const config = require("../src/config");
+let url = `https://storage.${config.getDomain()}:${config.getPort()}`;
+process.argv.push(`--url=${url}`);
+require("sliftutils/storage/remoteStorage/storageServerCli");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.525.0",
+  "version": "0.527.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -49,7 +49,8 @@
     "audit-imports": "./bin/audit-imports.js",
     "audit-disk-values": "./bin/audit-disk-values.js",
     "mcp-indexed-logs": "./bin/mcp-indexed-logs.js",
-    "autofix": "./bin/autofix.js"
+    "autofix": "./bin/autofix.js",
+    "storageserve": "./bin/storageserve.js"
   },
   "dependencies": {
     "@types/fs-ext": "^2.0.3",
@@ -70,7 +71,7 @@
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
     "sliftutils": "^1.7.5",
-    "socket-function": "^1.2.14",
+    "socket-function": "^1.2.15",
     "terser": "^5.31.0",
     "typenode": "^6.6.1",
     "typesafecss": "^0.32.0",

package/src/-b-authorities/cdnAuthority.ts

@@ -1,6 +1,6 @@
 import { Archives } from "../-a-archives/archives";
 import { ArchivesBackblaze } from "../-a-archives/archivesBackBlaze";
-import { cloudflareGETCall, cloudflarePOSTCall } from "./cloudflareHelpers";
+import { cloudflareGETCall, cloudflarePOSTCall } from "sliftutils/misc/https/cloudflareHelpers";
 import { setRecord, getZoneId } from "./dnsAuthority";
 import debugbreak from "debugbreak";
 

package/src/-b-authorities/cloudflareHelpers.ts

@@ -1,11 +1,9 @@
-import { httpsRequest } from "socket-function/src/https";
 import { getArchives } from "../-a-archives/archives";
 import { lazy } from "socket-function/src/caching";
-import { getStorageDir, getSubFolder } from "../fs";
+import { getStorageDir } from "../fs";
 import fs from "fs";
-import { isNodeTrue } from "socket-function/src/misc";
 
-export const keys = lazy(() => getArchives("keys"));
+const keys = lazy(() => getArchives("keys"));
 
 export const getCloudflareCreds = lazy(async (): Promise<{ key: string; email: string }> => {
     let credsJSON = await keys().get("cloudflare.json");
@@ -20,44 +18,3 @@ export const getCloudflareCreds = lazy(async (): Promise<{ key: string; email: s
     }
     return JSON.parse(credsJSON.toString());
 });
-
-export async function cloudflareGETCall<T>(path: string, params?: { [key: string]: string }): Promise<T> {
-    let url = new URL(`https://api.cloudflare.com/client/v4` + path);
-    for (let key in params) {
-        url.searchParams.set(key, params[key]);
-    }
-    let creds = await getCloudflareCreds();
-    let result = await httpsRequest(url.toString(), [], "GET", undefined, {
-        headers: {
-            "Content-Type": "application/json",
-            "X-Auth-Email": creds.email,
-            "X-Auth-User-Service-Key": creds.key,
-            "X-Auth-Key": creds.key,
-        }
-    });
-    let result2 = JSON.parse(result.toString()) as { result: unknown; success: boolean; errors: { code: number; message: string }[] };
-    if (!result2.success) {
-        throw new Error(`Cloudflare call failed: ${result2.errors.map(x => x.message).join(", ")}`);
-    }
-    return result2.result as T;
-}
-export async function cloudflarePOSTCall<T>(path: string, params: { [key: string]: unknown }): Promise<T> {
-    return await cloudflareCall(path, Buffer.from(JSON.stringify(params)), "POST");
-}
-export async function cloudflareCall<T>(path: string, payload: Buffer, method: string): Promise<T> {
-    let url = new URL(`https://api.cloudflare.com/client/v4` + path);
-    let creds = await getCloudflareCreds();
-    let result = await httpsRequest(url.toString(), payload, method, undefined, {
-        headers: {
-            "Content-Type": "application/json",
-            "X-Auth-Email": creds.email,
-            "X-Auth-User-Service-Key": creds.key,
-            "X-Auth-Key": creds.key,
-        }
-    });
-    let result2 = JSON.parse(result.toString()) as { result: unknown; success: boolean; errors: { code: number; message: string }[] };
-    if (!result2.success) {
-        throw new Error(`Cloudflare call failed: ${result2.errors.map(x => x.message).join(", ")}`);
-    }
-    return result2.result as T;
-}

package/src/-b-authorities/dnsAuthority.ts

@@ -1,185 +1,44 @@
-import os from "os";
-import * as fs from "fs";
-import { cache, lazy } from "socket-function/src/caching";
-import { isNode, isNodeTrue, timeInDay } from "socket-function/src/misc";
-import { httpsRequest } from "../https";
-import { getStorageDir } from "../fs";
-import { SocketFunction } from "socket-function/SocketFunction";
-import { delay } from "socket-function/src/batching";
-import debugbreak from "debugbreak";
-import { isClient } from "../config2";
-import { getArchives } from "../-a-archives/archives";
-import { cloudflareCall, cloudflareGETCall, cloudflarePOSTCall, getCloudflareCreds } from "./cloudflareHelpers";
-import { magenta } from "socket-function/src/formatting/logColors";
-
-const DNS_TTLSeconds = {
-    "TXT": 60,
-    "A": 60,
-};
-
-const DNS_REFRESH_STALE_AFTER = timeInDay;
-
-// We stamp our own "last asserted" time into the record's comment, because Cloudflare's
-//  modified_on can only be moved by an actual content change - and re-asserting an already
-//  correct record has no content change to make (a create errors with 81058, a no-op edit
-//  doesn't bump the timestamp). Reading freshness from text we control sidesteps that entirely,
-//  and lets us refresh a stale-but-correct record in place instead of deleting and recreating it.
-const FRESHNESS_REGEX = /<set on:[^>]*>/;
-
-/** Strips any prior freshness tag and appends a new one, preserving other comment text. */
-function stampFreshness(comment?: string): string {
-    let base = (comment ?? "").replace(FRESHNESS_REGEX, "").trim();
-    let stamp = `<set on: ${new Date().toString()}>`;
-    return base ? `${base} ${stamp}` : stamp;
-}
-/** Parses our tag back out; 0 (i.e. always stale) when it's absent or unparseable. */
-export function freshnessTime(comment?: string): number {
-    let match = FRESHNESS_REGEX.exec(comment ?? "");
-    if (!match) return 0;
-    return new Date(match[0].replace("<set on:", "").replace(">", "").trim()).getTime() || 0;
-}
-
-export const hasDNSWritePermissions = lazy(async () => {
-    if (!isNode()) return false;
-    if (isClient()) return false;
-    try {
-        await getCloudflareCreds();
-        return true;
-    } catch {
-        return false;
-    }
-});
-
-export const getZoneId = cache(async (rootDomain: string): Promise<string> => {
-    let zones = await cloudflareGETCall<{ id: string; name: string }[]>("/zones", {});
-    let selected = zones.find(x => x.name === rootDomain);
-    if (!selected) {
-        throw new Error(`Could not find zone for ${rootDomain}. Found ${zones.map(x => x.name).join(", ")}`);
-    }
-    return selected.id;
-});
-
-function getRootDomain(key: string) {
-    if (key.endsWith(".")) {
-        key = key.slice(0, -1);
-    }
-    return key.split(".").slice(-2).join(".");
-}
-
-export async function getRecordsRaw(type: string, key: string) {
-    if (key.endsWith(".")) key = key.slice(0, -1);
-    let zoneId = await getZoneId(getRootDomain(key));
-    let results = await cloudflareGETCall<{
-        id: string;
-        type: string;
-        name: string;
-        content: string;
-        proxied: boolean;
-        modified_on: string;
-        // Omitted by Cloudflare when the record has no comment.
-        comment?: string;
-    }[]>(`/zones/${zoneId}/dns_records`);
-    // DNS names are case-insensitive and Cloudflare returns them lowercased, so a mixed-case key
-    //  (e.g. a machine id subdomain) would never match an exact-case compare - match lowercased.
-    let keyLower = key.toLowerCase();
-    return results.filter(x => x.type === type && x.name.toLowerCase() === keyLower);
-}
-
-/** Cloudflare's batch endpoint applies deletes, then patches, then posts in a single database
- *   transaction. We route edits (patches) through here because the standalone PATCH/PUT verbs
- *   aren't usable in our setup, and because it lets "remove others + assert target" happen
- *   without a window where the name resolves to nothing. */
-export async function batchRecords(zoneId: string, batch: {
-    deletes?: { id: string }[];
-    patches?: { id: string; comment?: string }[];
-    posts?: { type: string; name: string; content: string; ttl: number; proxied: boolean; comment?: string }[];
-}) {
-    let payload: { [key: string]: unknown } = {};
-    if (batch.deletes && batch.deletes.length > 0) payload.deletes = batch.deletes;
-    if (batch.patches && batch.patches.length > 0) payload.patches = batch.patches;
-    if (batch.posts && batch.posts.length > 0) payload.posts = batch.posts;
-    try {
-        await cloudflarePOSTCall(`/zones/${zoneId}/dns_records/batch`, payload);
-    } catch (error) {
-        console.error(`Error updating DNS records:`, { error: error, batch });
-        throw new Error(`Error updating DNS records. ${JSON.stringify(batch)}. Error: ${error}`);
-    }
-}
-export async function getRecords(type: string, key: string) {
-    if (key.endsWith(".")) key = key.slice(0, -1);
-    let raw = await getRecordsRaw(type, key);
-    return raw.map(x => x.content);
-}
-export async function deleteRecord(type: string, key: string, value: string) {
-    if (key.endsWith(".")) key = key.slice(0, -1);
-    let zoneId = await getZoneId(getRootDomain(key));
-    let prevValues = await getRecordsRaw(type, key);
-    prevValues = prevValues.filter(x => x.content === value);
-    if (prevValues.length === 0) {
-        if (!SocketFunction.silent) {
-            console.log(`No need to delete record, it was not found. ${JSON.stringify(value)} value was not in ${type} for ${key}, values ${JSON.stringify(prevValues.map(x => x.content))}`);
-        }
-        return;
-    }
-
-    console.log(`Removing records of ${type} for ${key}, values ${JSON.stringify(prevValues.map(x => x.content))}`);
-    for (let value of prevValues) {
-        await cloudflareCall(`/zones/${zoneId}/dns_records/${value.id}`, Buffer.from([]), "DELETE");
-    }
-}
-/** Removes all existing records (unless the record is already present and fresh) */
-export async function setRecord(type: string, key: string, value: string, proxied?: "proxied", staleAfter = DNS_REFRESH_STALE_AFTER) {
-    if (key.endsWith(".")) key = key.slice(0, -1);
-    let zoneId = await getZoneId(getRootDomain(key));
-    let prevValues = await getRecordsRaw(type, key);
-    let existing = prevValues.find(x => x.content === value);
-    let others = prevValues.filter(x => x.content !== value);
-
-    // Already correct and recently asserted - a prior run also cleaned up the other records,
-    //  so there is nothing left to do.
-    if (existing && Date.now() - freshnessTime(existing.comment) < staleAfter) return;
-
-    // A single atomic batch: drop the wrong records, and either refresh the existing record's
-    //  comment in place or create it - so the name is never left resolving to nothing.
-    let ttl = DNS_TTLSeconds[type as "A"] || 60;
-    let comment = stampFreshness(existing?.comment);
-    console.log(magenta(`Setting ${type} record for ${key} to ${value} (previously had ${JSON.stringify(prevValues.map(x => x.content))})`));
-    await batchRecords(zoneId, {
-        deletes: others.map(x => ({ id: x.id })),
-        patches: existing ? [{ id: existing.id, comment }] : [],
-        posts: existing ? [] : [{ type, name: key, content: value, ttl, proxied: proxied === "proxied", comment }],
-    });
-    // Only a brand new record needs to propagate; an in-place comment refresh doesn't change the answer.
-    if (!existing) {
-        console.log(`Waiting ${ttl} seconds for DNS to propagate...`);
-        for (let ttlLeft = ttl; ttlLeft > 0; ttlLeft--) {
-            await delay(1000);
-            console.log(`${ttlLeft} seconds left...`);
-        }
-        console.log(`Done waiting for DNS to update.`);
-    }
-}
-/** Keeps existing records */
-export async function addRecord(type: string, key: string, value: string, proxied?: "proxied", staleAfter = DNS_REFRESH_STALE_AFTER) {
-    if (key.endsWith(".")) key = key.slice(0, -1);
-    let zoneId = await getZoneId(getRootDomain(key));
-    let prevValues = await getRecordsRaw(type, key);
-    let existing = prevValues.find(x => x.content === value);
-    if (existing && Date.now() - freshnessTime(existing.comment) < staleAfter) return;
-
-    // Same single-batch flow as setRecord, minus the deletes (we keep sibling records here).
-    let ttl = DNS_TTLSeconds[type as "A"] || 60;
-    let comment = stampFreshness(existing?.comment);
-    console.log(`Adding ${type} record for ${key} to ${value} (previously had ${JSON.stringify(prevValues.map(x => x.content))})`);
-    await batchRecords(zoneId, {
-        patches: existing ? [{ id: existing.id, comment }] : [],
-        posts: existing ? [] : [{ type, name: key, content: value, ttl, proxied: proxied === "proxied", comment }],
-    });
-    if (existing) return;
-    console.log(`Waiting ${ttl} seconds for DNS to propagate...`);
-    for (let ttlLeft = ttl; ttlLeft > 0; ttlLeft--) {
-        await delay(1000);
-        console.log(`${ttlLeft} seconds left...`);
-    }
-    console.log(`Done waiting for DNS to update.`);
-}
+import { lazy } from "socket-function/src/caching";
+import { isNode } from "socket-function/src/misc";
+import { isClient } from "../config2";
+import {
+    addRecord as addRecordBase,
+    deleteRecord as deleteRecordBase,
+    freshnessTime as freshnessTimeBase,
+    getRecords as getRecordsBase,
+    getZoneId as getZoneIdBase,
+    hasDNSWritePermissions as hasDNSWritePermissionsBase,
+    setRecord as setRecordBase,
+} from "sliftutils/misc/https/dns";
+
+export function freshnessTime(comment?: string): number {
+    return freshnessTimeBase(comment);
+}
+
+export const hasDNSWritePermissions = lazy(async () => {
+    if (!isNode()) return false;
+    if (isClient()) return false;
+    return await hasDNSWritePermissionsBase();
+});
+
+export async function getZoneId(rootDomain: string): Promise<string> {
+    return await getZoneIdBase(rootDomain);
+}
+
+export async function getRecords(type: string, key: string): Promise<string[]> {
+    return await getRecordsBase(type, key);
+}
+
+export async function deleteRecord(type: string, key: string, value: string): Promise<void> {
+    return await deleteRecordBase(type, key, value);
+}
+
+/** Removes all existing records (unless the record is already present and fresh) */
+export async function setRecord(type: string, key: string, value: string, proxied?: "proxied", staleAfter?: number): Promise<void> {
+    return await setRecordBase(type, key, value, proxied, staleAfter);
+}
+
+/** Keeps existing records */
+export async function addRecord(type: string, key: string, value: string, proxied?: "proxied", staleAfter?: number): Promise<void> {
+    return await addRecordBase(type, key, value, proxied, staleAfter);
+}

package/src/-c-identity/IdentityController.ts

@@ -251,6 +251,7 @@ const changeIdentityOnce = cacheWeak(async function changeIdentityOnce(connectio
         () => new Error(`Timeout calling changeIdentity for ${nodeId}`)
     );
 });
+let startupTime = Date.now();
 SocketFunction.addGlobalClientHook(async function identityHook(context) {
     if (context.call.classGuid === IdentityController._classGuid) return;
     // This is for US to tell them our identity. And if they established the connection the identity will come from their original connection url (that they used to connect to us), and they validated it either being a real certificate, or they added the cert from the trusted backblaze bucket. If it just from a real certificate it means we identified them, but they might not have network trust. But that's fine, as IdentityController is JUST for identification, and if it's a real certificate we know who they are! (Which doesn't mean we trust them).
@@ -259,8 +260,9 @@ SocketFunction.addGlobalClientHook(async function identityHook(context) {
     }
     let time = Date.now();
     await changeIdentityOnce(context.connectionId);
-    let duration = Date.now() - time;
-    if (duration > 200) {
+    let now = Date.now();
+    let duration = now - time;
+    if (duration > 200 && now - startupTime > timeInMinute) {
         console.log(red(`IdentityHook took ${formatTime(duration)} for ${context.connectionId.nodeId} ${context.call.classGuid}.${context.call.functionName}`));
     }
 });

package/src/-d-trust/NetworkTrust2.ts

@@ -119,7 +119,7 @@ export const loadServerCert = cache(async (machineId: string) => {
         console.warn(`Could not find certificate in archives for ${machineId}`);
         return;
     }
-    console.log(magenta(`Loading certificate for ${machineId}`));
+    console.info(magenta(`Loading certificate for ${machineId}`));
     trustCertificate(certFile);
 });
 

package/src/-f-node-discovery/LatencyTracking.ts

@@ -8,7 +8,7 @@
 */
 
 import { SocketFunction } from "socket-function/SocketFunction";
-import { timeInMinute, timeInSecond } from "socket-function/src/misc";
+import { sort, timeInMinute, timeInSecond } from "socket-function/src/misc";
 import { lazy } from "socket-function/src/caching";
 import { delay, runInfinitePollCallAtStart } from "socket-function/src/batching";
 import { isServer } from "../config2";
@@ -23,6 +23,8 @@ const LATENCY_POLL_INTERVAL = timeInMinute;
 const PING_TIMEOUT = timeInSecond * 5;
 // Rolling average size. Small, so latency changes show up quickly.
 const LATENCY_SAMPLE_LIMIT = 20;
+// Raw sample history kept per node, so callers can ask for a median over the last N with a known confidence.
+const LATENCY_HISTORY_LIMIT = 100;
 // Forget nodes we haven't been able to reach for this long.
 const NODE_EXPIRY_TIME = LATENCY_POLL_INTERVAL * 5;
 
@@ -30,6 +32,7 @@ export type NodeLatencyInfo = {
     averageLatency: number;
     sampleCount: number;
     lastSeen: number;
+    history: number[];
 };
 
 // otherNodeId => our rolling latency to it
@@ -42,6 +45,14 @@ export function getNodeLatencyInfo(nodeId: string): NodeLatencyInfo | undefined
 export function getNodeLatency(nodeId: string): number | undefined {
     return getNodeLatencyInfo(nodeId)?.averageLatency;
 }
+// Median of the last historyCount raw samples, plus how many samples were actually available — so callers can scale how much they trust the number.
+export function getNodeLatencyMedian(config: { nodeId: string; historyCount: number }): { latency: number; historyUsed: number } | undefined {
+    let history = getNodeLatencyInfo(config.nodeId)?.history;
+    if (!history || history.length === 0) return undefined;
+    let samples = history.slice(-config.historyCount);
+    sort(samples, x => x);
+    return { latency: samples[Math.floor(samples.length / 2)], historyUsed: samples.length };
+}
 
 export function getCachedNodeLatencyInfoList(): Map<string, NodeLatencyInfo> {
     return latencyByNode;
@@ -60,10 +71,16 @@ export function getOwnLatencies(): { [nodeId: string]: number } {
 function recordLatency(nodeId: string, latency: number) {
     let prev = latencyByNode.get(nodeId);
     let sampleCount = Math.min(prev?.sampleCount || 0, LATENCY_SAMPLE_LIMIT);
+    let history = prev?.history || [];
+    history.push(latency);
+    if (history.length > LATENCY_HISTORY_LIMIT) {
+        history.shift();
+    }
     latencyByNode.set(nodeId, {
         averageLatency: ((prev?.averageLatency || 0) * sampleCount + latency) / (sampleCount + 1),
         sampleCount: sampleCount + 1,
         lastSeen: Date.now(),
+        history,
     });
 }
 

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -216,7 +216,7 @@ async function setNodeIds(nodeIds: string[]) {
     if (newNodeIds.length === 0 && removedNodeIds.length === 0) return;
     if (logging) {
         for (let nodeId of newNodeIds) {
-            console.log(blue(`Discovered node ${nodeId}`));
+            console.info(blue(`Discovered node ${nodeId}`));
         }
         for (let nodeId of removedNodeIds) {
             console.log(red(`Removed node from setNodeIds ${nodeId}`));

package/src/-h-path-value-serialize/PathValueSerializer.ts

@@ -20,7 +20,7 @@ import { formatNumber, formatPercent, formatTime } from "socket-function/src/for
 setFlag(require, "cbor-x", "allowclient", true);
 
 import * as pako from "pako";
-import { delay } from "socket-function/src/batching";
+import { delay, safeLoop } from "socket-function/src/batching";
 import { LZ4 } from "../storage/LZ4";
 import { unblockLoop } from "socket-function/src/batching";
 setFlag(require, "pako", "allowclient", true);
@@ -345,6 +345,7 @@ class PathValueSerializer {
         let encodedValues: (Buffer | number)[] = [];
 
         let i = 0;
+        let valuesSinceYield = 0;
         for (let values of valuesGroups) {
             measureBlock(() => {
                 // NOTE: Writing one value at a time is about 1.6X slower. BUT, it allows us to very efficient decode
@@ -378,21 +379,23 @@ class PathValueSerializer {
                     i++;
                 }
             }, "valuesWriteLoop");
-            await delay("paintLoop");
+            valuesSinceYield += values.length;
+            if (valuesSinceYield >= 10_000) {
+                valuesSinceYield = 0;
+                await delay("paintLoop");
+            }
         }
 
         measureBlock(function valuesWriteTypesBuffer() {
             writer.writeBuffer(types);
         });
-        // Break encodeValues into groups of 100, so we can check the time and delay
-        //      if we are taking too long, but not check it EVERY loop, as that would
-        //      be too slow.
+        // Break encodeValues into groups, and yield based on the amount of values written — NOT by measuring time, as timing every loop is itself too slow.
         let valueGroups: (Buffer | number)[][] = [];
         const VALUE_GROUP_SIZE = 1000;
         for (let i = 0; i < encodedValues.length; i += VALUE_GROUP_SIZE) {
             valueGroups.push(encodedValues.slice(i, i + VALUE_GROUP_SIZE));
         }
-        let prevTime = Date.now();
+        let encodedSinceYield = 0;
         for (let encodedValues of valueGroups) {
             measureBlock(function valuesWriteTypes() {
                 for (let value of encodedValues) {
@@ -403,10 +406,10 @@ class PathValueSerializer {
                     }
                 }
             });
-            let now = Date.now();
-            if (now - prevTime > 10) {
+            encodedSinceYield += encodedValues.length;
+            if (encodedSinceYield >= 10_000) {
+                encodedSinceYield = 0;
                 await delay("paintLoop");
-                prevTime = now;
             }
         }
     }
@@ -550,9 +553,14 @@ class PathValueSerializer {
         }
 
         startNewBuffer("pathValues");
+        let pathValuesSinceYield = 0;
         for (let values of valueGroups) {
             this.pathValuesWrite(writer, values, settings);
-            await delay("afterPaint");
+            pathValuesSinceYield += values.length;
+            if (pathValuesSinceYield >= 10_000) {
+                pathValuesSinceYield = 0;
+                await delay("afterPaint");
+            }
         }
 
         if (!settings.noLocks) {
@@ -593,15 +601,20 @@ class PathValueSerializer {
             curCount += str.length;
         }
 
+        let stringBytesSinceYield = 0;
         for (let strings of stringParts) {
             let stringBuffer = StringSerialize.serializeStrings(strings);
             settings.bufferMap!.push("strings");
             outputBuffers.push(stringBuffer);
-            await delay("paintLoop");
+            stringBytesSinceYield += stringBuffer.length;
+            if (stringBytesSinceYield >= 1_000_000) {
+                stringBytesSinceYield = 0;
+                await delay("paintLoop");
+            }
         }
 
         if (settings.compression === "lz4") {
-            let compressedBuffers = await unblockLoop(outputBuffers, x => LZ4.compress(x));
+            let compressedBuffers = await measureBlock(() => safeLoop({ data: outputBuffers, name: "PathValueSerializer.serialize|lz4" }, x => LZ4.compress(x)), "PathValueSerializer.serialize|lz4");
 
             // If the compress factor is less than a threshold, use the uncompressed buffers
             let uncompressedSize = outputBuffers.reduce((total, x) => total + x.length, 0);
@@ -723,6 +736,7 @@ class PathValueSerializer {
         }
         if (!config?.skipStrings) {
             let stringArrays: string[][] = [];
+            let bytesSinceYield = 0;
             for (let stringBuffer of stringBuffers) {
                 let obj = StringSerialize.deserializeStringsLazy(stringBuffer);
                 while (true) {
@@ -731,11 +745,14 @@ class PathValueSerializer {
                         break;
                     }
                     stringArrays.push(nextStrings);
-                    if (stringArrays.length > 1) {
+                    // Each getNextStrings call decodes ~1MB, so on buffers past that we yield per chunk decoded.
+                    if (stringBuffer.length > 1_000_000) {
                         await delay("paintLoop");
                     }
                 }
-                if (stringBuffers.length > 1) {
+                bytesSinceYield += stringBuffer.length;
+                if (bytesSinceYield >= 1_000_000) {
+                    bytesSinceYield = 0;
                     await delay("paintLoop");
                 }
             }
@@ -848,6 +865,7 @@ class PathValueSerializer {
             }
 
             let stringArrays: string[][] = [];
+            let bytesSinceYield = 0;
             for (let bufferIndex of stringBufferIndexes) {
                 let stringBuffer = buffers[bufferIndex];
                 let obj = StringSerialize.deserializeStringsLazy(stringBuffer);
@@ -857,13 +875,12 @@ class PathValueSerializer {
                         break;
                     }
                     stringArrays.push(nextStrings);
-                    if (stringArrays.length > 1) {
+                    bytesSinceYield += stringBuffer.length;
+                    if (bytesSinceYield >= 1_000_000) {
+                        bytesSinceYield = 0;
                         await delay("paintLoop");
                     }
                 }
-                if (stringBufferIndexes.length > 1) {
-                    await delay("paintLoop");
-                }
             }
             strings = stringArrays.flat();
         }

package/src/0-path-value-core/AuthorityLookup.ts

@@ -1,4 +1,4 @@
-import { timeInMinute, timeInSecond } from "socket-function/src/misc";
+import { sort, timeInMinute, timeInSecond } from "socket-function/src/misc";
 import { nestArchives } from "../-a-archives/archives";
 import { getArchivesBackblaze } from "../-a-archives/archivesBackBlaze";
 import { archiveJSONT } from "../-a-archives/archivesJSONT";
@@ -31,7 +31,7 @@ let CALL_TIMEOUT = timeInSecond * 5;
 const SYNC_JITTER_WINDOW = timeInSecond * 10;
 
 // Nodes this close to their scheduled shutdown are avoided as sources (only used if nothing else can satisfy the request).
-const SHUTDOWN_AVOID_WINDOW = timeInMinute * 10;
+const SHUTDOWN_AVOID_WINDOW = timeInMinute * 5;
 
 export type AuthorityEntry = {
     nodeId: string;
@@ -82,6 +82,8 @@ class AuthorityLookup {
     public async setOurSpec(spec: AuthoritySpec) {
         if (!SocketFunction.isMounted()) throw new Error("Cannot call setOurPaths without mounting first (use Querysub.hostService).");
         spec.nodeId = getOwnNodeId();
+        // AuthoritySpec.prefixes promises to be sorted, and this is the choke point every published spec goes through.
+        sort(spec.prefixes, prefix => prefix.originalPrefix);
 
 
         if (this.setSpec) {