querysub 0.590.0 → 0.592.0

package/src/4-deploy/edgeNodes.ts

@@ -3,14 +3,12 @@ import { getOwnMachineId } from "sliftutils/misc/https/certs";
 import { devDebugbreak, getDomain, isLocal, isPublic, noSyncing } from "../config";
 import child_process from "child_process";
 import { getAllNodeIds, getOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
-import { getArchivesBackblazePublic } from "../-a-archives/archivesBackBlaze";
-import { nestArchives } from "../-a-archives/archives";
+import { getArchives2Public } from "../-a-archives/archives2";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { delay, runInSerial, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
 import { compare, compareArray, isNodeTrue, sort, timeInMinute, timeInSecond } from "socket-function/src/misc";
 import { cacheLimited, lazy } from "socket-function/src/caching";
 import { canHaveChildren } from "socket-function/src/types";
-import { hostArchives } from "../-b-authorities/cdnAuthority";
 import { getModuleFromConfig } from "../3-path-functions/pathFunctionLoader";
 import path from "path";
 import debugbreak from "debugbreak";
@@ -31,7 +29,7 @@ import type { FunctionRunnerIndex } from "../4-querysub/FunctionRunnerTracking";
 const UPDATE_POLL_INTERVAL = timeInMinute * 15;
 const DEAD_NODE_COUNT_THRESHOLD = 15;
 
-const edgeNodeStorage = isNodeTrue() && nestArchives("edgenodes/", getArchivesBackblazePublic(getDomain()));
+const edgeNodeStorage = isNodeTrue() && getArchives2Public("edgenodes/");
 const edgeNodeIndexFile = "edge-nodes-index.json";
 
 const getEdgeNodeConfig = cacheLimited(10000, async (fileName: string): Promise<EdgeNodeConfig | undefined> => {
@@ -265,19 +263,15 @@ async function publishOwnScheduledShutdown(time: number) {
     await updateEdgeNodesFile();
 }
 
-export const getEdgeNodeConfigURL = lazy(async () => {
-    let { getURL } = await hostArchives({
-        archives: edgeNodeStorage,
-        subdomain: `edge`,
-        domain: getDomain(),
-    });
-    return await getURL(edgeNodeIndexFile);
-});
+/** Every public URL that can serve the edge node index (write node first, then fastest-first). Not cached: the underlying source config can change (scheduled window switchovers), and after the one initialization await this is synchronous anyway. */
+export async function getEdgeNodeConfigURLs(): Promise<string[]> {
+    return await edgeNodeStorage.getURLs(edgeNodeIndexFile);
+}
 
 const startUpdateLoop = lazy(async () => {
     startEdgeNotifier();
     SocketFunction.expose(EdgeNodeController);
-    await getEdgeNodeConfigURL();
+    await getEdgeNodeConfigURLs();
     await runInfinitePollCallAtStart(UPDATE_POLL_INTERVAL * (1 + Math.random() * 0.1), updateLoop);
 });
 

package/src/4-querysub/Querysub.ts

@@ -825,9 +825,9 @@ export class Querysub {
 
         RequireController.injectHTMLBeforeStartup(async () => {
             const { getEdgeBootstrapScript } = await import("../4-deploy/edgeBootstrap");
-            const { getEdgeNodeConfigURL } = await import("../4-deploy/edgeNodes");
+            const { getEdgeNodeConfigURLs } = await import("../4-deploy/edgeNodes");
             let edgeBootstrapFile = await getEdgeBootstrapScript({
-                edgeNodeConfigURL: await getEdgeNodeConfigURL(),
+                edgeNodeConfigURLs: await getEdgeNodeConfigURLs(),
             });
             return `<script>${edgeBootstrapFile}</script>`;
         });

package/src/config2.ts

@@ -1,10 +1,4 @@
-import { deepCloneJSON, isNode } from "socket-function/src/misc";
-import { hasArchivesPermissions } from "./-a-archives/archives";
-import { baseIsClient, getDomain } from "./config";
-import { JSONLACKS } from "socket-function/src/JSONLACKS/JSONLACKS";
-import { rootPathStr, prependToPathStr, getPathDepth } from "./path";
-import fs from "fs";
-import { AuthoritySpec } from "./0-path-value-core/PathRouter";
+import { baseIsClient } from "./config";
 
 export function isClient() {
     return baseIsClient();

package/src/deployManager/components/ServiceMeasureBars.tsx

@@ -2,7 +2,8 @@ import { SocketFunction } from "socket-function/SocketFunction";
 import { qreact } from "../../4-dom/qreact";
 import { css } from "typesafecss";
 import { formatNumber, formatTime, formatVeryNiceDateTime } from "socket-function/src/formatting/format";
-import { timeoutToUndefinedSilent, timeInSecond } from "socket-function/src/misc";
+import { timeInSecond } from "socket-function/src/misc";
+import { timeoutToError } from "../../errors";
 import { StatsValue, getStatsTop } from "socket-function/src/profiling/stats";
 import type { MeasureChunk, MeasureSummary } from "../../diagnostics/watchdog";
 import { NodeCapabilitiesController } from "../../-g-core-values/NodeCapabilities";
@@ -10,19 +11,19 @@ import { getSyncedController } from "../../library-components/SyncedController";
 import { assertIsManagementUser } from "../../diagnostics/managementPages";
 
 const NODE_CALL_TIMEOUT = timeInSecond * 10;
-const BAR_HEIGHT_PX = 10;
-const MIN_SEGMENT_WIDTH_PX = 30;
+const BAR_HEIGHT_PX = 6;
+const MIN_BAR_WIDTH_PX = 30;
+// Green at idle, red at fully busy.
+const BAR_IDLE_HUE = 120;
+const BAR_FLASH_THRESHOLD = 0.6;
+const FLASH_CLASS_NAME = "ServiceMeasureBars-flash";
 
 class ServiceMeasureControllerBase {
-    public async getMeasureSummaries(nodeIds: string[]): Promise<Record<string, MeasureSummary | undefined>> {
-        let result: Record<string, MeasureSummary | undefined> = {};
-        await Promise.all(nodeIds.map(async nodeId => {
-            result[nodeId] = await timeoutToUndefinedSilent(NODE_CALL_TIMEOUT, NodeCapabilitiesController.nodes[nodeId].getMeasureSummary());
-        }));
-        return result;
+    public async getMeasureSummary(nodeId: string): Promise<MeasureSummary | undefined> {
+        return await timeoutToError(NODE_CALL_TIMEOUT, NodeCapabilitiesController.nodes[nodeId].getMeasureSummary(), () => new Error(`Timed out asking ${nodeId} for its measure summary`));
     }
     public async getMeasureBreakdown(nodeId: string, range: { startTime: number; endTime: number }): Promise<{ chunk?: MeasureChunk }> {
-        let chunk = await timeoutToUndefinedSilent(NODE_CALL_TIMEOUT, NodeCapabilitiesController.nodes[nodeId].getMeasureBreakdown(range));
+        let chunk = await timeoutToError(NODE_CALL_TIMEOUT, NodeCapabilitiesController.nodes[nodeId].getMeasureBreakdown(range), () => new Error(`Timed out asking ${nodeId} for its measure breakdown`));
         return { chunk };
     }
 }
@@ -30,7 +31,7 @@ export const ServiceMeasureController = getSyncedController(SocketFunction.regis
     "ServiceMeasureController-3f8c1de2-5b74-4e0f-a6c9-8d21f47b90a5",
     new ServiceMeasureControllerBase(),
     () => ({
-        getMeasureSummaries: { hooks: [assertIsManagementUser], compress: true },
+        getMeasureSummary: { hooks: [assertIsManagementUser], compress: true },
         getMeasureBreakdown: { hooks: [assertIsManagementUser], compress: true },
     }),
     () => ({
@@ -41,52 +42,55 @@ function percent(value: number) {
     return `${(value * 100).toFixed(2)}%`;
 }
 
-function nameHue(name: string) {
-    let hash = 0;
-    for (let i = 0; i < name.length; i++) {
-        hash = (hash * 31 + name.charCodeAt(i)) | 0;
-    }
-    return Math.abs(hash) % 360;
-}
-
 export class ServiceMeasureBars extends qreact.Component<{
     serviceNodeIds: string[];
-    /** Every node id on the page, so all rows share one cached call. */
-    allNodeIds: string[];
     expandedNodeId: string;
     onToggleNode: (nodeId: string) => void;
 }> {
     render() {
         let controller = ServiceMeasureController(SocketFunction.browserNodeId());
-        let summaries = controller.getMeasureSummaries(this.props.allNodeIds);
-        if (!summaries) return undefined;
 
-        return <div className={css.vbox(2)}>
+        return <div className={css.vbox(1).alignItems("stretch")}>
             {this.props.serviceNodeIds.map(nodeId => {
-                let summary = summaries[nodeId];
+                let isExpanded = this.props.expandedNodeId === nodeId;
+                let summary: MeasureSummary | undefined;
+                try {
+                    summary = controller.getMeasureSummary(nodeId);
+                } catch (err) {
+                    return <div key={nodeId}
+                        className={css.height(BAR_HEIGHT_PX).hsl(0, 70, 55)}
+                        title={`Node did not respond (it likely doesn't have the measure API yet)\n${nodeId}\n${(err as Error).stack ?? err}`}
+                    />;
+                }
                 if (!summary) return undefined;
                 let timeRunFor = summary.endTime - summary.startTime;
                 if (timeRunFor <= 0) return undefined;
-                let isExpanded = this.props.expandedNodeId === nodeId;
-                return <div key={nodeId}
-                    className={
-                        css.hbox(1).wrap.button
-                        + (isExpanded && css.outline("1px solid hsl(210, 60%, 50%)") || "")
-                    }
-                    onClick={() => this.props.onToggleNode(nodeId)}
-                >
-                    {summary.entries.map(entry => {
-                        let fraction = entry.ownTime / timeRunFor;
-                        return <div key={entry.name}
-                            className={css
-                                .height(BAR_HEIGHT_PX)
+                let fraction = summary.profiledTime / timeRunFor;
+                let clampedFraction = Math.min(1, Math.max(0, fraction));
+                let hue = BAR_IDLE_HUE * (1 - clampedFraction);
+                let isFlashing = fraction > BAR_FLASH_THRESHOLD;
+                return <div key={nodeId} className={css.hbox(0)}>
+                    <div
+                        className={
+                            css.height(BAR_HEIGHT_PX)
                                 .width(`${fraction * 100}%`)
-                                .minWidth(MIN_SEGMENT_WIDTH_PX)
-                                .background(`hsl(${nameHue(entry.name)}, 60%, 60%)`)
-                            }
-                            title={`${entry.name}\n${formatTime(entry.ownTime)} (${percent(fraction)} of ${formatTime(timeRunFor)} profiled)\n${nodeId}`}
-                        />;
-                    })}
+                                .minWidth(MIN_BAR_WIDTH_PX)
+                                .button
+                                .hsl(hue, 70, isExpanded ? 40 : 55)
+                            + (isFlashing && (" " + FLASH_CLASS_NAME) || "")
+                        }
+                        title={`Profiled ${formatTime(summary.profiledTime)} of ${formatTime(timeRunFor)} (${percent(fraction)} CPU)\n${nodeId}`}
+                        onClick={() => this.props.onToggleNode(nodeId)}
+                    />
+                    {isFlashing && <style>{`
+                        @keyframes ${FLASH_CLASS_NAME}-anim {
+                            0%, 100% { opacity: 1; }
+                            50% { opacity: 0.3; }
+                        }
+                        .${FLASH_CLASS_NAME} {
+                            animation: ${FLASH_CLASS_NAME}-anim 0.8s infinite;
+                        }
+                    `}</style>}
                 </div>;
             })}
         </div>;
@@ -95,15 +99,21 @@ export class ServiceMeasureBars extends qreact.Component<{
 
 export class ServiceMeasureBreakdown extends qreact.Component<{
     nodeId: string;
-    allNodeIds: string[];
 }> {
     render() {
         let controller = ServiceMeasureController(SocketFunction.browserNodeId());
-        let summaries = controller.getMeasureSummaries(this.props.allNodeIds);
-        let summary = summaries?.[this.props.nodeId];
-        if (!summary) return undefined;
-
-        let result = controller.getMeasureBreakdown(this.props.nodeId, { startTime: summary.startTime, endTime: summary.endTime });
+        let summary: MeasureSummary | undefined;
+        let result: { chunk?: MeasureChunk } | undefined;
+        try {
+            summary = controller.getMeasureSummary(this.props.nodeId);
+            if (!summary) return undefined;
+            result = controller.getMeasureBreakdown(this.props.nodeId, { startTime: summary.startTime, endTime: summary.endTime });
+        } catch (err) {
+            return <div className={css.pad2(10).hsl(0, 70, 92).bord2(0, 0, 20)}>
+                <div className={css.boldStyle.colorhsl(0, 70, 35)}>Node did not respond (it likely doesn't have the measure API yet)</div>
+                <pre className={css.whiteSpace("pre-wrap").fontSize(12)}>{(err as Error).stack ?? String(err)}</pre>
+            </div>;
+        }
         if (!result) {
             return <div className={css.pad2(10).hsl(0, 0, 98).bord2(0, 0, 20)}>Loading breakdown...</div>;
         }

package/src/deployManager/components/ServicesListPage.tsx

@@ -17,10 +17,12 @@ import { PendingDeployInfo, UpdateButtons, UpdateServiceButtons } from "./deploy
 import { isDefined } from "../../misc";
 import { formatDateJSX } from "../../misc/formatJSX";
 import { Tools } from "./Tools";
-import { ServiceMeasureBars, ServiceMeasureBreakdown } from "./ServiceMeasureBars";
+import { ServiceMeasureBars, ServiceMeasureBreakdown, ServiceMeasureController } from "./ServiceMeasureBars";
 
 module.hotreload = true;
 
+const INVALIDATE_FLASH_TIME = timeInSecond;
+
 /** What kind of service this is, when we can tell from its command, linking to wherever that kind is administered. Its own box beside the service's, so clicking the service still opens the service. */
 class ServiceCategoryBadge extends qreact.Component<{ config: ServiceConfig }> {
     render() {
@@ -45,6 +47,7 @@ export class ServicesListPage extends qreact.Component {
     state = t.state({
         measurementsEnabled: t.atomic<boolean>(true),
         expandedMeasureNodes: t.lookup(t.string),
+        invalidateClickTime: t.atomic<number>(0),
     });
 
     render() {
@@ -81,7 +84,6 @@ export class ServicesListPage extends qreact.Component {
                 serviceNodeIds.set(serviceId, nodeIds);
             }
         }
-        let allNodeIds = sort(Array.from(new Set(Array.from(serviceNodeIds.values()).flat())), x => x);
 
         return <div className={css.vbox(16)}>
             <div className={css.hbox(12).wrap}>
@@ -124,13 +126,29 @@ export class ServicesListPage extends qreact.Component {
                 </button>
                 <UpdateButtons services={services.map(x => x[1]).filter(isDefined)} />
             </div>
-            <div
-                className={css.pad2(12, 8).button.bord2(0, 0, 20).hsl(0, 0, 95).alignSelf("flex-start")}
-                onClick={() => {
-                    this.state.measurementsEnabled = !this.state.measurementsEnabled;
-                }}
-            >
-                <span className={css.boldStyle}>{this.state.measurementsEnabled ? "☑" : "☐"} Measurements</span>
+            <div className={css.hbox(8)}>
+                <div
+                    className={css.pad2(12, 8).button.bord2(0, 0, 20).hsl(0, 0, 95)}
+                    onClick={() => {
+                        this.state.measurementsEnabled = !this.state.measurementsEnabled;
+                    }}
+                >
+                    <span className={css.boldStyle}>{this.state.measurementsEnabled ? "☑" : "☐"} Measurements</span>
+                </div>
+                {this.state.measurementsEnabled && (() => {
+                    let justInvalidated = now < this.state.invalidateClickTime + INVALIDATE_FLASH_TIME;
+                    return <div
+                        className={css.pad2(12, 8).button.bord2(0, 0, 20)
+                            + (justInvalidated && css.hsl(120, 50, 80) || css.hsl(0, 0, 95))
+                        }
+                        onClick={() => {
+                            this.state.invalidateClickTime = Date.now();
+                            ServiceMeasureController.resetAll();
+                        }}
+                    >
+                        {justInvalidated ? "Invalidated ✓" : "Invalidate"}
+                    </div>;
+                })()}
             </div>
             <Tools />
             <div className={css.vbox(8)}>
@@ -168,10 +186,10 @@ export class ServicesListPage extends qreact.Component {
                     let duplicateKey = (keyCounts.get(config.parameters.key || "") || 0) > 1;
                     let measureNodeIds = serviceNodeIds.get(serviceId) || [];
                     let expandedMeasureNodeId = this.state.expandedMeasureNodes[serviceId] || "";
-                    return <div className={css.vbox(4)} key={serviceId}>
+                    return <div className={css.vbox(4).alignItems("stretch")} key={serviceId}>
                         <div className={css.hbox(10)}>
                         <ServiceCategoryBadge config={config} />
-                        <div className={css.vbox(4)}>
+                        <div className={css.vbox(4).alignItems("stretch")}>
                         <Anchor noStyles
                             values={[currentViewParam.getOverride("service-detail"), selectedServiceIdParam.getOverride(serviceId)]}
                             className={
@@ -240,7 +258,6 @@ export class ServicesListPage extends qreact.Component {
                         </Anchor>
                         {this.state.measurementsEnabled && measureNodeIds.length > 0 && <ServiceMeasureBars
                             serviceNodeIds={measureNodeIds}
-                            allNodeIds={allNodeIds}
                             expandedNodeId={expandedMeasureNodeId}
                             onToggleNode={nodeId => {
                                 if (this.state.expandedMeasureNodes[serviceId] === nodeId) {
@@ -256,7 +273,6 @@ export class ServicesListPage extends qreact.Component {
                         </div>
                         {this.state.measurementsEnabled && expandedMeasureNodeId && <ServiceMeasureBreakdown
                             nodeId={expandedMeasureNodeId}
-                            allNodeIds={allNodeIds}
                         />}
                     </div>
                         ;

package/src/deployManager/machineDaemonShared.ts

@@ -0,0 +1,2 @@
+export const SERVICE_NAME = "machine-alwaysup";
+export const SERVICE_UNIT_NAME = `${SERVICE_NAME}.service`;

package/src/deployManager/machineSchema.ts

@@ -1,6 +1,5 @@
 import { isNodeTrue, list, timeInDay, timeInMinute, timeInSecond } from "socket-function/src/misc";
-import { nestArchives } from "../-a-archives/archives";
-import { getArchivesBackblaze } from "../-a-archives/archivesBackBlaze";
+import { getArchives2 } from "../-a-archives/archives2";
 import { getDomain } from "../config";
 import { archiveJSONT } from "../-a-archives/archivesJSONT";
 import { SocketFunction } from "socket-function/SocketFunction";
@@ -187,9 +186,9 @@ export type MachineConfig = {
     machineId: string;
     disabled: boolean;
 };
-export const machineInfos = archiveJSONT<MachineInfo>(() => nestArchives("machines/machine-heartbeats/", getArchivesBackblaze(getDomain())));
-export const serviceConfigs = archiveJSONT<ServiceConfig>(() => nestArchives("machines/service-configs/", getArchivesBackblaze(getDomain())));
-export const machineConfigs = archiveJSONT<MachineConfig>(() => nestArchives("machines/machine-configs/", getArchivesBackblaze(getDomain())));
+export const machineInfos = archiveJSONT<MachineInfo>(() => getArchives2("machines/machine-heartbeats/"));
+export const serviceConfigs = archiveJSONT<ServiceConfig>(() => getArchives2("machines/service-configs/"));
+export const machineConfigs = archiveJSONT<MachineConfig>(() => getArchives2("machines/machine-configs/"));
 
 export type LaunchRecord = {
     serviceId: string;
@@ -207,7 +206,7 @@ export type LaunchSummary = {
     key: string;
 };
 
-const launches = lazy(() => nestArchives("machines/launches/", getArchivesBackblaze(getDomain())));
+const launches = lazy(() => getArchives2("machines/launches/"));
 
 function formatLaunchDay(time: number): string {
     let date = new Date(time);

package/src/deployManager/setupMachineMain.ts

@@ -8,13 +8,12 @@ import readline from "readline";
 import open from "open";
 import { fsExistsAsync } from "../fs";
 import { delay } from "socket-function/src/batching";
+import { SERVICE_NAME, SERVICE_UNIT_NAME } from "./machineDaemonShared";
 // Import querysub, to fix missing dependencies
 Querysub;
 
 const pinnedNodeVersion = 22;
 
-const SERVICE_NAME = "machine-alwaysup";
-const SERVICE_UNIT_NAME = `${SERVICE_NAME}.service`;
 const DAEMON_SCRIPT_NAME = "machine-daemon.sh";
 // How long after the process dies before it is started again. Short, because until it is back nothing on the machine is being deployed or repaired.
 const SERVICE_RESTART_DELAY_SECONDS = 5;

package/src/deployManager/stopMachineMain.ts

@@ -0,0 +1,45 @@
+import { runPromise } from "../functional/runCommand";
+import { SERVICE_NAME, SERVICE_UNIT_NAME } from "./machineDaemonShared";
+
+const DPLY_SCREEN_SUFFIX = "-dply";
+
+async function main() {
+    let sshRemote = process.argv.slice(2).join(" ");
+    if (!sshRemote) {
+        console.error("Incorrect usage. Examples:\nyarn stop-machine 153.34.64.2\nyarn stop-machine devops@153.34.64.2");
+        process.exit(1);
+    }
+
+    await runPromise(`ssh ${sshRemote} whoami`);
+
+    let daemonState = (await runPromise(`ssh ${sshRemote} "systemctl is-active ${SERVICE_UNIT_NAME} || true"`, { nothrow: true })).trim();
+    if (daemonState === "active") {
+        console.log(`Stopping the ${SERVICE_UNIT_NAME} daemon...`);
+        await runPromise(`ssh ${sshRemote} "sudo systemctl stop ${SERVICE_UNIT_NAME}"`);
+        console.log(`✅ Daemon stopped`);
+    } else {
+        console.log(`✅ The ${SERVICE_UNIT_NAME} daemon is not running (state: ${daemonState || "unknown"})`);
+    }
+
+    let sessions = (await runPromise(`ssh ${sshRemote} "tmux list-sessions -F '#{session_name}' 2>/dev/null || true"`, { nothrow: true }))
+        .split("\n").map(x => x.trim()).filter(x => x);
+    // The machine's own screen first, so it cannot restart a service screen while the rest are being killed
+    let toKill = [
+        ...sessions.filter(name => name === SERVICE_NAME),
+        ...sessions.filter(name => name.endsWith(DPLY_SCREEN_SUFFIX)),
+    ];
+    if (toKill.length === 0) {
+        console.log(`✅ No ${SERVICE_NAME} or *${DPLY_SCREEN_SUFFIX} tmux sessions to kill (sessions: ${sessions.join(", ") || "none"})`);
+    }
+    for (let name of toKill) {
+        console.log(`Killing tmux session ${name}...`);
+        await runPromise(`ssh ${sshRemote} "tmux kill-session -t ${name}"`);
+    }
+    if (toKill.length > 0) {
+        console.log(`✅ Killed ${toKill.length} tmux session(s): ${toKill.join(", ")}`);
+    }
+
+    console.log(`\n🛑 Machine stopped. Start it again with: ssh ${sshRemote} "~/machine-startup.sh"`);
+}
+
+main().catch(console.error).finally(() => process.exit(0));

package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts

@@ -5,15 +5,17 @@ import { BufferIndex } from "./BufferIndex";
 import { delay, runInParallel, runInSerial, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
 import { IndexedLogResults, Reader, SearchParams, addReadToResults, createEmptyIndexedLogResults, INDEX_EXTENSION, mergeIndexedLogResults } from "./BufferIndexHelpers";
 import { getDomain, isPublic } from "../../../config";
-import { getArchivesHome, getArchivesLocal } from "../../../-a-archives/archivesDisk";
-import { getArchivesBackblaze, getArchivesBackblazePrivateImmutable } from "../../../-a-archives/archivesBackBlaze";
+import { getArchivesHome } from "../../../-a-archives/archivesDisk";
+import { getArchives2 } from "../../../-a-archives/archives2";
 import { getMachineId, getOwnThreadId } from "sliftutils/misc/https/certs";
 import { ArchivesMemoryCacheStats, createArchivesMemoryCache } from "../../../-a-archives/archivesMemoryCache";
+import { createArchivesMemoryCache2 } from "../../../-a-archives/archivesMemoryCache2";
+import { wrapArchivesWithCache2 } from "../../../-a-archives/archiveCache2";
 import { registerShutdownHandler } from "../../periodic";
 import { measureBlock, measureFnc, measureWrap } from "socket-function/src/profiling/measure";
 import { isNode } from "typesafecss";
 import { getOwnMachineId, isNodeIdOnOwnMachineId, isOwnNodeId } from "../../../-f-node-discovery/NodeDiscovery";
-import { TimeFilePath, TimeFileTree } from "./TimeFileTree";
+import { LogArchives, TimeFilePath, TimeFileTree } from "./TimeFileTree";
 import { LogStreamer } from "./LogStreamer";
 import { moveLogsToPublic } from "./moveIndexLogsToPublic";
 import { MAX_SINGLE_FILE_DATA, MAX_COUNT_PER_FILE, DISK_FLUSH_INTERVAL, PUBLIC_MOVE_THRESHOLD, MOVING_TIMEOUT } from "./BufferIndexLogsOptimizationConstants";
@@ -83,7 +85,7 @@ export class IndexedLogs<T> {
 
     private getLocalLogs = lazy((): Archives => {
         let baseDisk = getArchivesHome(getDomain());
-        let archives = nestArchives("indexed-logs/" + this.config.name, baseDisk);
+        let archives = nestArchives("indexed-logs/" + this.config.name, baseDisk) as Archives;
         archives = createArchivesMemoryCache(archives, {
             maxSize: 1024 * 1024 * 512,
             maxCount: 1000 * 100,
@@ -94,32 +96,43 @@ export class IndexedLogs<T> {
         });
         return archives;
     });
-    private getPublicLogsBase(loadPublic: boolean): Archives {
-        let basePublic: Archives = getArchivesHome(getDomain());
-        // NOTE: The local disk is so fast that reading in 10 megabytes is nothing, And if we read in too small of a value, the overhead per read ends up making this take forever. 
-        let extraReadSize = 1024 * 1024 * 10;
+    private getPublicLogsBase(loadPublic: boolean): LogArchives {
         if (loadPublic) {
-            basePublic = getArchivesBackblaze(getDomain());
-            // NOTE: While the latency to back plays is high, now we're reading in parallel, so it shouldn't be as big of an issue. 
-            extraReadSize = 1024 * 1024 * 1;
+            return createArchivesMemoryCache2(getArchives2("final-indexed-logs/" + this.config.name), {
+                maxSize: 1024 * 1024 * 1024 * 2,
+                maxCount: 1000 * 500,
+                fullyImmutable: true,
+                // NOTE: While the latency to the remote is high, now we're reading in parallel, so it shouldn't be as big of an issue.
+                extraReadSize: 1024 * 1024 * 1,
+                stats: this.backblazeLogsStats,
+                sizeCache: this.fileSizeCache,
+            });
         }
-        let archives = nestArchives("final-indexed-logs/" + this.config.name, basePublic);
-        archives = createArchivesMemoryCache(archives, {
+        let archives = nestArchives("final-indexed-logs/" + this.config.name, getArchivesHome(getDomain()));
+        return createArchivesMemoryCache(archives, {
             maxSize: 1024 * 1024 * 1024 * 2,
             maxCount: 1000 * 500,
             fullyImmutable: true,
-            extraReadSize,
+            // NOTE: The local disk is so fast that reading in 10 megabytes is nothing, And if we read in too small of a value, the overhead per read ends up making this take forever.
+            extraReadSize: 1024 * 1024 * 10,
             stats: this.backblazeLogsStats,
             sizeCache: this.fileSizeCache,
         });
-        return archives;
     };
     public debugGetCachedLogs = cache((config: {
         type: "local" | "public";
-    }) => {
-        let usePublic = config.type === "public";
-        let archives = usePublic ? getArchivesBackblaze(getDomain()) : getArchivesHome(getDomain());
-        archives = nestArchives("final-indexed-logs/" + this.config.name, archives);
+    }): LogArchives => {
+        if (config.type === "public") {
+            let archives2 = wrapArchivesWithCache2(getArchives2("final-indexed-logs/" + this.config.name), {
+                immutable: true,
+            });
+            return createArchivesMemoryCache2(archives2, {
+                maxSize: 1024 * 1024 * 1024 * 12,
+                maxCount: 1000 * 500,
+                fullyImmutable: true,
+            });
+        }
+        let archives = nestArchives("final-indexed-logs/" + this.config.name, getArchivesHome(getDomain())) as Archives;
         archives = wrapArchivesWithCache(archives, {
             immutable: true,
         });
@@ -133,10 +146,10 @@ export class IndexedLogs<T> {
     public debugIsPublic() {
         return isPublic();
     }
-    private getPublicLogs = lazy((): Archives => {
+    private getPublicLogs = lazy((): LogArchives => {
         return this.getPublicLogsBase(isPublic());
     });
-    private getRealPublicLogs = lazy((): Archives => {
+    private getRealPublicLogs = lazy((): LogArchives => {
         return this.getPublicLogsBase(true);
     });
 

package/src/diagnostics/logs/IndexedLogs/MCPIndexedLogs.ts

@@ -5,7 +5,7 @@ import { getDomain } from "../../../config";
 import { getAllNodeIds, getOwnMachineId, isOwnNodeId } from "../../../-f-node-discovery/NodeDiscovery";
 import { NodeCapabilitiesController } from "../../../-g-core-values/NodeCapabilities";
 import { getLoggers2Async, LogDatum } from "../diskLogger";
-import { Archives } from "../../../-a-archives/archives";
+import { LogArchives } from "./TimeFileTree";
 import { TimeFilePath, TimeFileTree } from "./TimeFileTree";
 import { BufferReader, INDEX_EXTENSION, createEmptyIndexedLogResults } from "./BufferIndexHelpers";
 import { BufferIndex } from "./BufferIndex";
@@ -104,7 +104,7 @@ type Direction = "fromStart" | "fromEnd";
 type FileEntry = {
     path: TimeFilePath;
     loggerName: LoggerName;
-    archives: Archives;
+    archives: LogArchives;
 };
 
 // Mutable accumulators the file/block walk appends into, instead of returning
@@ -570,7 +570,7 @@ export class MCPIndexedLogs {
     // Caches TimeFileTree.findAllPaths by hour-aligned bucket so repeated
     // searches over similar windows reuse the slow folder walk.
     private async getCachedPaths(config: {
-        archives: Archives;
+        archives: LogArchives;
         type: "local" | "public";
         loggerName: LoggerName;
         startTime: number;

package/src/diagnostics/logs/IndexedLogs/TimeFileTree.ts

@@ -1,7 +1,16 @@
 import { timeInDay } from "socket-function/src/misc";
-import { Archives } from "../../../-a-archives/archives";
 import { getOwnThreadId, getOwnMachineId } from "../../../-f-node-discovery/NodeDiscovery";
 
+/** The subset of archive operations the log system uses, so both the old disk Archives and archives2 (IArchives) work interchangeably. */
+export type LogArchives = {
+    getDebugName(): string;
+    get(path: string, config?: { range?: { start: number; end: number; } }): Promise<Buffer | undefined>;
+    set(path: string, data: Buffer): Promise<unknown>;
+    del(path: string): Promise<void>;
+    getInfo(path: string): Promise<{ writeTime: number; size: number; } | undefined>;
+    find(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<string[]>;
+};
+
 export type TimeFilePath = {
     fullPath: string;
 
@@ -113,7 +122,7 @@ export function decodeLogFilePath(path: string): TimeFilePath | undefined {
 
 // NOTE: This class will add its own extensions to the files, and it'll use it to detect these files. You can append your own extension and then we will ignore those files. 
 export class TimeFileTree {
-    public constructor(public archives: Archives) { }
+    public constructor(public archives: LogArchives) { }
 
     public async findAllPaths(config: {
         startTime: number;

package/src/diagnostics/logs/IndexedLogs/moveIndexLogsToPublic.ts

@@ -1,10 +1,9 @@
 import { delay } from "socket-function/src/batching";
 import { sort, keyByArray } from "socket-function/src/misc";
-import { Archives } from "../../../-a-archives/archives";
 import { getOwnThreadId } from "../../../-f-node-discovery/NodeDiscovery";
 import { BufferIndex } from "./BufferIndex";
 import { LogStreamer } from "./LogStreamer";
-import { TimeFileTree } from "./TimeFileTree";
+import { LogArchives, TimeFileTree } from "./TimeFileTree";
 import { blue, green, magenta } from "socket-function/src/formatting/logColors";
 import { formatDateTime, formatNumber, formatTime } from "socket-function/src/formatting/format";
 
@@ -17,8 +16,8 @@ export async function moveLogsToPublic(config: {
     movingTimeout: number;
     // Unsafe, but useful for testing
     forceAll: boolean;
-    localLogs: Archives;
-    publicLogs: Archives;
+    localLogs: LogArchives;
+    publicLogs: LogArchives;
     indexExtension: string;
 }) {
     let { forceAll, localLogs, publicLogs, publicMoveThreshold, maxSingleFileData, indexExtension, movingTimeout } = config;

package/src/diagnostics/logs/errorTickets/tickets.ts

@@ -2,10 +2,9 @@ import { lazy } from "socket-function/src/caching";
 import { batchFunction, runInfinitePollCallAtStart } from "socket-function/src/batching";
 import { timeInMinute } from "socket-function/src/misc";
 import { SocketFunction } from "socket-function/SocketFunction";
-import { nestArchives } from "../../../-a-archives/archives";
-import { getArchivesBackblaze } from "../../../-a-archives/archivesBackBlaze";
+import { getArchives2 } from "../../../-a-archives/archives2";
 import { archiveJSONT } from "../../../-a-archives/archivesJSONT";
-import { getDomain, isPublic } from "../../../config";
+import { isPublic } from "../../../config";
 import { getControllerNodeId } from "../../../-g-core-values/NodeCapabilities";
 import { assertIsManagementUser } from "../../managementPages";
 import { getSyncedController } from "../../../library-components/SyncedController";
@@ -14,7 +13,7 @@ import { Ticket, TicketComment, TicketPatchStatus, TicketState } from "./ticketT
 
 const TICKET_CACHE_POLL_INTERVAL = timeInMinute * 5;
 
-export const ticketsArchive = archiveJSONT<Ticket>(() => nestArchives("logs/error-tickets/", getArchivesBackblaze(getDomain())));
+export const ticketsArchive = archiveJSONT<Ticket>(() => getArchives2("logs/error-tickets/"));
 
 let ticketCache = new Map<string, Ticket>();
 let ensureWatching = lazy(async () => {