querysub 0.199.0 → 0.201.0

package/bin/machine-alwaysup.js

@@ -1,4 +1,3 @@
 #!/usr/bin/env node
 
-require("typenode");
-require("../src/deployManager/applyAlwaysUpMain");
+require("../src/deployManager/applyAlwaysUpMain.js");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.199.0",
+  "version": "0.201.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/-0-hooks/hooks.ts

@@ -12,13 +12,13 @@ function createHookFunction<Fnc extends (...args: any[]) => void>(debugName: str
 } {
     let queuedCalls = [] as Parameters<Fnc>[];
     let declaration: Fnc | undefined;
-    setImmediate(() => {
+    setTimeout(() => {
         if (!declaration) {
             setTimeout(flushQueued, 1000);
             return;
         }
         flushQueued();
-    });
+    }, 1000);
     function flushQueued() {
         if (!declaration) {
             throw new Error(`Hook function ${debugName} not declared`);

package/src/0-path-value-core/pathValueCore.ts

@@ -912,8 +912,9 @@ class AuthorityPathValueStorage {
     public async onShutdown() {
         if (yargObj.noarchive) return;
         this.shuttingDown.resolve("shutdown");
+        if (this.pendingArchiveValues.size === 0) return;
         await pathValueAuthority2.waitUntilRoutingIsReady();
-        // If there are other nodes, we CAN'T just archive our values, as some may be rejected in the future.
+        // If there are other nodes, we CAN'T just archive our values, as some maybe rejected in the future.
         let nodes = pathValueAuthority2.getReadNodes(rootPathStr);
         let otherReadNodes = nodes.filter(x => !isOwnNodeId(x));
         if (otherReadNodes.length > 0 && this.pendingArchiveValues.size > 0) {

package/src/4-deploy/git.ts

@@ -23,6 +23,7 @@ export const setGitRef = measureWrap(async function setGitRef(config: {
     repoUrl: string;
     gitRef: string;
 }) {
+    await fs.promises.mkdir(config.gitFolder, { recursive: true });
     let hostKey = await runPromise(`ssh-keyscan -t rsa bitbucket.org`);
     hostKey = hostKey.split("\n").filter(x => !x.startsWith("#")).join("\n");
     let knownHostsPath = os.homedir() + "/.ssh/known_hosts";

package/src/deployManager/MachinesPage.tsx

@@ -15,8 +15,8 @@ export class MachinesPage extends qreact.Component {
             //.bord2(0, 0, 20)
         }>
             {[
+                { key: "machines", label: "Machines", otherKeys: ["machine-detail"] },
                 { key: "services", label: "Services", otherKeys: ["service-detail"] },
-                { key: "machines", label: "Machines", otherKeys: ["machine-detail"] }
             ].map(tab => {
                 let isActive = currentViewParam.value === tab.key || tab.otherKeys.includes(currentViewParam.value);
                 return <div key={tab.key}

package/src/deployManager/applyAlwaysUpMain.js

@@ -1,7 +1,8 @@
 const child_process = require("child_process");
 const fs = require("fs");
+const os = require("os");
 
-export function shellEscape(array) {
+function shellEscape(array) {
     return array
         .map((value) => {
             if (/[^A-Za-z0-9_\/:=-]/.test(value)) {
@@ -50,23 +51,22 @@ async function runCommandWithLogging(command) {
         });
 
         childProc.on("error", (err) => {
-            reject(combinedOutput.slice(-1000));
+            reject(`Always up machine process error: ${err}\nRecent output:${combinedOutput.slice(-1000)}`);
         });
 
         childProc.on("close", (code) => {
-            reject(`${combinedOutput.slice(-1000)}\nProcess exited with code ${code}, recent output:`);
+            reject(`Always up machine process exited with code ${code}, recent output:${combinedOutput.slice(-1000)}`);
         });
     });
 }
 
 async function main() {
-    let args = process.argv.slice(2);
     while (true) {
         try {
-            await runCommandWithLogging(shellEscape(args));
+            await runCommandWithLogging("yarn machine");
         } catch (e) {
             console.error(`Process finished, restarting in 5 seconds`);
-            fs.writeFileSync("lastAlwaysUpError.txt", e.stack);
+            await fs.promises.writeFile(os.homedir() + "/lastAlwaysUpError.txt", e?.stack || e);
         }
         await delay(5000);
     }

package/src/deployManager/components/MachinesListPage.tsx

@@ -52,7 +52,6 @@ export class MachinesListPage extends qreact.Component {
                         <button
                             className={css.pad2(12, 8).button.bord2(0, 0, 20)
                                 + (hasSelectedMachines ? css.hsl(0, 80, 90) : css.hsl(0, 0, 80))}
-                            disabled={!hasSelectedMachines || this.state.isDeleting}
                             onClick={() => {
                                 if (!hasSelectedMachines) return;
 
@@ -111,14 +110,13 @@ export class MachinesListPage extends qreact.Component {
 
                     return <div key={machineId}
                         className={
-                            css.pad2(12).bord2(0, 0, 20)
-                            + (this.state.isDeleteMode ? "" : css.button)
+                            css.pad2(12).bord2(0, 0, 20).button
                             + (
                                 failingServices.length > 0 && css.hsl(0, 50, 60)
                                 || isMachineDead && css.hsl(0, 0, 50)
                                 || css.hsl(0, 0, 100)
                             )
-                            + (this.state.isDeleteMode && isSelected ? css.bord2(200, 80, 60, 2) : "")
+                            + (this.state.isDeleteMode && isSelected && css.bord2(200, 80, 60, 2))
                         }
                         onClick={() => {
                             if (this.state.isDeleteMode) {

package/src/deployManager/components/ServicesListPage.tsx

@@ -69,11 +69,15 @@ export class ServicesListPage extends qreact.Component {
                         let serviceInfo = machineInfo?.services[serviceId];
                         return serviceInfo && !serviceInfo.errorFromLastRun;
                     });
+                    let missingMachines = config.machineIds.filter(machineId => {
+                        let machineInfo = getMachineInfo(machineId);
+                        return !machineInfo?.services[serviceId];
+                    });
                     let totalLaunches = config.machineIds.reduce((acc, machineId) => {
                         let machineInfo = getMachineInfo(machineId);
                         return acc + (machineInfo?.services[serviceId].totalTimesLaunched || 0);
                     }, 0);
-                    let notFailingNotRunning = config.machineIds.length - runningMachines.length - failingMachines.length;
+                    let unknown = config.machineIds.length - runningMachines.length - failingMachines.length - missingMachines.length;
                     return <div key={serviceId}
                         className={
                             css.pad2(12).button.bord2(0, 0, 20)
@@ -92,7 +96,7 @@ export class ServicesListPage extends qreact.Component {
                                 <div className={css.fontSize(14).boldStyle}>{config.info.title}</div>
                                 <div>{config.parameters.key}</div>
                                 <div>
-                                    {config.machineIds.length} configured {failingMachines.length > 0 && `(${failingMachines.length} failing)` || notFailingNotRunning > 0 && `(${notFailingNotRunning} not fail and not running)`} • {totalLaunches} launches •
+                                    {config.machineIds.length} configured {failingMachines.length > 0 && `(${failingMachines.length} failing)`} {missingMachines.length > 0 && `(${missingMachines.length} missing machines)`} {unknown > 0 && `(${unknown} unknown)`} • {totalLaunches} launches •
                                     Deploy: {config.parameters.deploy ? "enabled" : "disabled"}
                                 </div>
                             </div>

package/src/deployManager/machine.sh

@@ -0,0 +1,3 @@
+#! /bin/bash
+tmux new -s machine-alwaysup -d
+tmux send-keys -t machine-alwaysup "cd ~/machine-alwaysup && yarn machine-alwaysup" Enter

package/src/deployManager/machineApplyMainCode.ts

@@ -321,17 +321,8 @@ export async function machineApplyMain() {
     // Wait for the console to get shimmed
     await delay(1000);
 
-    if (true) {
-        console.log(`Dying in a bit`);
-        setTimeout(() => {
-            void shutdown();
-        }, 10000);
-        return;
-    }
-
-    console.error("Apply special error test");
-
-    let lastErrorPath = "lastAlwaysUpError.txt";
+    // NOTE: Error's don't get logged unless we host, so... we can't do this earlier than this...
+    let lastErrorPath = os.homedir() + "/lastAlwaysUpError.txt";
     if (fs.existsSync(lastErrorPath)) {
         let lastError = await fs.promises.readFile(lastErrorPath, "utf8");
         await fs.promises.unlink(lastErrorPath);
@@ -351,6 +342,7 @@ export async function machineApplyMain() {
         }
     }
     await fs.promises.writeFile(isRunningPath, `${process.ppid}-${process.pid}`);
+
     await Querysub.hostService("machine-apply");
     onServiceConfigChange(resyncServices);
 }

package/src/deployManager/setupMachineMain.ts

@@ -1,3 +1,309 @@
-//todonext
-// The only arg is the remoteIP (and optionally username, I guess...)
-// Run a lot of the commands remotely, one at a time. I guess make a runSSH helper function
+import { getBackblazePath } from "../-a-archives/archivesBackBlaze";
+import { getGitURLLive, getGitRefLive } from "../4-deploy/git";
+import { Querysub } from "../4-querysub/QuerysubController";
+import { runPromise } from "../functional/runCommand";
+import fs from "fs";
+import os from "os";
+import readline from "readline";
+import open from "open";
+// Import querysub, to fix missing dependencies
+Querysub;
+
+const pinnedNodeVersion = 22;
+
+async function getGitHubApiKey(repoUrl: string, sshRemote: string): Promise<string> {
+    // Parse repository info from URL
+    let repoOwner = "";
+    let repoName = "";
+    // Handle both SSH and HTTPS formats
+    // SSH: git@github.com:owner/repo.git
+    // HTTPS: https://github.com/owner/repo.git
+    const sshMatch = repoUrl.match(/git@github\.com:([^/]+)\/(.+)\.git$/);
+    const httpsMatch = repoUrl.match(/https:\/\/github\.com\/([^/]+)\/(.+)\.git$/);
+
+    if (sshMatch) {
+        repoOwner = sshMatch[1];
+        repoName = sshMatch[2];
+    } else if (httpsMatch) {
+        repoOwner = httpsMatch[1];
+        repoName = httpsMatch[2];
+    }
+
+    const cacheFile = os.homedir() + `/githubkey_${repoOwner}_${repoName}.json`;
+
+    // Check if we have a cached key
+    if (fs.existsSync(cacheFile)) {
+        try {
+            const cached = JSON.parse(fs.readFileSync(cacheFile, "utf8"));
+            if (cached.apiKey) {
+                console.log(`✅ Using cached GitHub API key from ${cacheFile}`);
+                return cached.apiKey;
+            }
+        } catch {
+            // Invalid cache file, we'll ask for a new key
+        }
+    }
+
+    // Need to get a new API key from user
+    console.log("\n🔑 GitHub API key required for private repository access");
+    console.log("Opening GitHub token creation page...");
+
+    // Construct URL for classic token (fine-grained tokens don't support deploy keys)
+    const repoInfo = repoOwner && repoName ? ` for repository ${repoOwner}/${repoName}` : "";
+    const instructions = `yarn setup-machine ${sshRemote}${repoInfo}
+1) Set expiration to 'No expiration' 
+2) Check the 'repo' scope (full repository access)
+3) Click 'Generate token'`;
+
+    let tokenUrl = `https://github.com/settings/tokens/new?description=${encodeURIComponent(instructions)}&scopes=repo`;
+    if (repoOwner && repoName) {
+        console.log(`Setting up access for repository: ${repoOwner}/${repoName}`);
+    }
+
+    await open(tokenUrl);
+
+    const rl = readline.createInterface({
+        input: process.stdin,
+        output: process.stdout
+    });
+
+    const apiKey = await new Promise<string>((resolve) => {
+        const repoInfo = repoOwner && repoName ? ` for repository ${repoOwner}/${repoName}` : "";
+        rl.question(`Please paste your GitHub API token (${repoInfo}): `, (answer) => {
+            rl.close();
+            resolve(answer.trim());
+        });
+    });
+
+    // Cache the key
+    fs.writeFileSync(cacheFile, JSON.stringify({ apiKey }));
+    console.log(`✅ Caching GitHub API key to ${cacheFile}`);
+
+    return apiKey;
+}
+
+async function addDeployKeyToGitHub(sshPublicKey: string, keyTitle: string, repoUrl: string, sshRemote: string): Promise<void> {
+    const apiKey = await getGitHubApiKey(repoUrl, sshRemote);
+
+    // Parse repository info from URL to get owner/repo for the API endpoint
+    let repoOwner = "";
+    let repoName = "";
+    const sshMatch = repoUrl.match(/git@github\.com:([^/]+)\/(.+)\.git$/);
+    const httpsMatch = repoUrl.match(/https:\/\/github\.com\/([^/]+)\/(.+)\.git$/);
+
+    if (sshMatch) {
+        repoOwner = sshMatch[1];
+        repoName = sshMatch[2];
+    } else if (httpsMatch) {
+        repoOwner = httpsMatch[1];
+        repoName = httpsMatch[2];
+    } else {
+        throw new Error(`Could not parse GitHub repository from URL: ${repoUrl}`);
+    }
+
+    let url = `https://api.github.com/repos/${repoOwner}/${repoName}/keys`;
+    console.log(url);
+    const response = await fetch(url, {
+        method: "POST",
+        headers: {
+            "Authorization": `Bearer ${apiKey}`,
+            "Accept": "application/vnd.github+json",
+            "Content-Type": "application/json"
+        },
+        body: JSON.stringify({
+            title: keyTitle,
+            key: sshPublicKey,
+            read_only: true
+        })
+    });
+
+    if (!response.ok) {
+        const errorText = await response.text();
+        throw new Error(`Failed to add deploy key to GitHub repository: ${response.status} ${errorText}`);
+    }
+
+    console.log("✅ Deploy key added to GitHub repository");
+}
+
+async function setupRepositoryOnRemote(sshRemote: string, gitURLLive: string, gitRefLive: string): Promise<void> {
+    // Create git folder on remote
+    await runPromise(`ssh ${sshRemote} "mkdir -p ~/machine-alwaysup"`);
+
+    // Add bitbucket.org host key to remote machine's known_hosts
+    await runPromise(`ssh ${sshRemote} "mkdir -p ~/.ssh"`);
+    await runPromise(`ssh ${sshRemote} "ssh-keyscan -t rsa bitbucket.org >> ~/.ssh/known_hosts 2>/dev/null || true"`);
+    await runPromise(`ssh ${sshRemote} "ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts 2>/dev/null || true"`);
+
+    // Check if repo already exists, if not clone it
+    try {
+        await runPromise(`ssh ${sshRemote} "cd ~/machine-alwaysup && git status"`);
+        console.log("Repository already exists, updating...");
+        // Repository exists, update it
+        await runPromise(`ssh ${sshRemote} "cd ~/machine-alwaysup && git remote update"`);
+        await runPromise(`ssh ${sshRemote} "cd ~/machine-alwaysup && git add --all"`);
+        await runPromise(`ssh ${sshRemote} "cd ~/machine-alwaysup && git stash"`);
+        await runPromise(`ssh ${sshRemote} "cd ~/machine-alwaysup && git fetch --all"`);
+        await runPromise(`ssh ${sshRemote} "cd ~/machine-alwaysup && git reset --hard ${gitRefLive}"`);
+        await runPromise(`ssh ${sshRemote} "cd ~/machine-alwaysup && git prune"`);
+    } catch {
+        console.log("Cloning repository...");
+        // Repository doesn't exist, clone it
+        await runPromise(`ssh ${sshRemote} "git clone ${gitURLLive} ~/machine-alwaysup"`);
+        await runPromise(`ssh ${sshRemote} "cd ~/machine-alwaysup && git reset --hard ${gitRefLive}"`);
+    }
+}
+
+async function main() {
+    let sshRemote = process.argv.slice(2).join(" ");
+    if (!sshRemote) {
+        console.error("Incorrect usage. Examples:\nyarn setup-machine 153.34.64.2\nyarn setup-machine devops@153.34.64.2");
+        process.exit(1);
+    }
+
+    // Test command to verify ssh credentials work
+    await runPromise(`ssh ${sshRemote} whoami`);
+
+    let backblazePath = getBackblazePath();
+
+    console.log("Setting up machine:", sshRemote);
+
+    // 1. Copy backblaze file to remote server (~/backblaze.json)
+    console.log("Copying backblaze credentials...");
+    if (fs.existsSync(backblazePath)) {
+        await runPromise(`scp "${backblazePath}" ${sshRemote}:~/backblaze.json`);
+        console.log("✅ Backblaze credentials copied");
+    } else {
+        console.warn("⚠️  Backblaze file not found at:", backblazePath);
+    }
+
+    // 2. Ensure git is installed on remote server
+    console.log("Ensuring git is installed...");
+    try {
+        await runPromise(`ssh ${sshRemote} "which git"`);
+        console.log("✅ Git already installed");
+    } catch {
+        console.log("Installing git...");
+        await runPromise(`ssh ${sshRemote} "sudo apt update && sudo apt install -y git"`);
+        console.log("✅ Git installed");
+    }
+
+    // 3. Ensure build tools are installed (needed for native modules)
+    console.log("Ensuring build tools are installed...");
+    try {
+        await runPromise(`ssh ${sshRemote} "which make"`);
+        console.log("✅ Build tools already installed");
+    } catch {
+        console.log("Installing build tools...");
+        await runPromise(`ssh ${sshRemote} "sudo apt install -y build-essential"`);
+        console.log("✅ Build tools installed");
+    }
+
+    // 3. Ensure nodejs is installed on remote server
+    console.log("Ensuring Node.js is installed...");
+    try {
+        let nodeVersion = await runPromise(`ssh ${sshRemote} "node --version"`);
+        console.log("✅ Node.js already installed:", nodeVersion.trim());
+    } catch {
+        console.log(`Installing Node.js (v${pinnedNodeVersion})...`);
+        await runPromise(`ssh ${sshRemote} "curl -fsSL https://deb.nodesource.com/setup_${pinnedNodeVersion}.x | sudo -E bash -"`);
+        await runPromise(`ssh ${sshRemote} "sudo apt-get install -y nodejs"`);
+        let nodeVersion = await runPromise(`ssh ${sshRemote} "node --version"`);
+        console.log("✅ Node.js installed:", nodeVersion.trim());
+    }
+
+    // 4. Ensure yarn is installed on remote server
+    console.log("Ensuring yarn is installed...");
+    try {
+        await runPromise(`ssh ${sshRemote} "which yarn"`);
+        console.log("✅ Yarn already installed");
+    } catch {
+        console.log("Installing yarn...");
+        await runPromise(`ssh ${sshRemote} "npm install -g yarn"`);
+        console.log("✅ Yarn installed");
+    }
+
+    // 5. Clone current repo into ~/machine-alwaysup with SSH key handling for private repos
+    console.log("Setting up repository...");
+    let gitURLLive = await getGitURLLive();
+    let gitRefLive = await getGitRefLive();
+
+    try {
+        await setupRepositoryOnRemote(sshRemote, gitURLLive, gitRefLive);
+        console.log("✅ Repository cloned and set to correct reference");
+    } catch (error) {
+        // Check if this is a private repository access issue
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        if (errorMessage.includes("Permission denied") || errorMessage.includes("Repository not found") || errorMessage.includes("fatal: Could not read from remote repository")) {
+            console.log("⚠️  Repository appears to be private, setting up SSH key access...");
+
+            // Ensure SSH key exists on remote machine
+            await runPromise(`ssh ${sshRemote} "mkdir -p ~/.ssh"`);
+            try {
+                await runPromise(`ssh ${sshRemote} "test -f ~/.ssh/id_rsa"`);
+                console.log("✅ SSH key already exists on remote machine");
+            } catch {
+                console.log("Generating SSH key on remote machine...");
+                await runPromise(`ssh ${sshRemote} "ssh-keygen -t rsa -b 4096 -f ~/.ssh/id_rsa -N ''"`);
+                console.log("✅ SSH key generated on remote machine");
+            }
+
+            // Get the public key from remote machine
+            const sshPublicKey = await runPromise(`ssh ${sshRemote} "cat ~/.ssh/id_rsa.pub"`);
+            const keyTitle = `Machine Setup - ${sshRemote} - ${new Date().toISOString()}`;
+
+            // Add the deploy key to GitHub repository
+            await addDeployKeyToGitHub(sshPublicKey.trim(), keyTitle, gitURLLive, sshRemote);
+
+            // Retry repository setup
+            console.log("Retrying repository setup with SSH key...");
+            await setupRepositoryOnRemote(sshRemote, gitURLLive, gitRefLive);
+            console.log("✅ Repository cloned and set to correct reference");
+        } else {
+            // Re-throw other errors
+            throw error;
+        }
+    }
+
+    // Install dependencies
+    console.log("Installing dependencies...");
+    await runPromise(`ssh ${sshRemote} "cd ~/machine-alwaysup && yarn install"`);
+    console.log("✅ Dependencies installed");
+
+    // 6. Create ~/machine-startup.sh which runs ~/machine-alwaysup/src/deployManager/machine.sh, and setup crontab
+    console.log("Setting up cron job...");
+
+    // Create startup script directly on remote machine
+    await runPromise(`ssh ${sshRemote} "echo '#!/bin/bash' > ~/machine-startup.sh"`);
+    await runPromise(`ssh ${sshRemote} "echo 'cd ~/machine-alwaysup/node_modules/querysub/src/deployManager' >> ~/machine-startup.sh"`);
+    await runPromise(`ssh ${sshRemote} "echo 'bash machine.sh' >> ~/machine-startup.sh"`);
+    await runPromise(`ssh ${sshRemote} "chmod +x ~/machine-startup.sh"`);
+
+    // 7. Setup crontab to run ~/machine-startup.sh on startup
+    const cronEntry = "@reboot ~/machine-startup.sh";
+
+    // Get existing crontab, add our entry if not already present
+    let existingCron = "";
+    try {
+        existingCron = await runPromise(`ssh ${sshRemote} "crontab -l"`, { nothrow: true });
+    } catch {
+        // No existing crontab is fine
+    }
+
+    if (!existingCron.includes("machine-startup.sh")) {
+        // Add cron entry directly
+        await runPromise(`ssh ${sshRemote} "(crontab -l 2>/dev/null || true; echo '${cronEntry}') | crontab -"`);
+        console.log("✅ Cron job added");
+    } else {
+        console.log("✅ Cron job already exists");
+    }
+
+    // Start the machine service immediately
+    console.log("Starting machine service...");
+    await runPromise(`ssh ${sshRemote} "~/machine-startup.sh"`);
+    console.log("✅ Machine service started!");
+
+    console.log("\n🎉 Machine setup complete!");
+}
+
+
+main().catch(console.error).finally(() => process.exit(0));

package/src/deployManager/spec.txt

@@ -1,10 +1,3 @@
-3) Enable console shimming, so machineApplyMain gets error notifications
-4) Verify applyMain terminating gets logged correctly
-
-
-4) Always up wrapper
-    - Reruns apply utility when it crashes
-    - And error log forwarding support, so crashes get logged by the next apply script to run, by writing apply utility crashes to a special file the apply utility reads on startup
 4.1) Requisition script
     - yarn setup-machine 153.34.64.2
 
@@ -23,10 +16,11 @@
             setup crontab to startup.sh
             run startup.sh
     - We might need to have it use to github API to give the remote machine access to the repo?
-    - Test on a new server, that we run temporarily with some test scripts
 
-4.1) Requisition script in "bin", as a .js bootstrapper
-    - Verify it works, so we can do 
+4) Test on a new server, that we run temporarily with some test scripts
+
+5) OH! Also setup swap usage on the machine, as it isn't by default often?
+
 
 
 
@@ -35,7 +29,8 @@
         - Hmm... requisition is not usable, as it is too hard to type. But something...
 
 5) Setup on our regular digital ocean server
-    - Remove previous startup script and kill existing tmux services
+    - Remove previous startup.sh, and crontab and kill existing tmux services
+
 6) Verify crash logging works with error notifications (it work if we just apply our console logs shims)
 7) Quick node removal on process crash or removal
     Detect the nodeId of services (if they have one), and when the service dies, immediately remove "edgenodes/" file, and trigger an update of "edge-nodes-index.json"

package/src/deployManager/urlParams.ts

@@ -3,6 +3,6 @@ import { URLParam } from "../library-components/URLParam";
 export type ViewType = "services" | "machines" | "service-detail" | "machine-detail";
 
 // URL Parameters for navigation state
-export const currentViewParam = new URLParam<ViewType>("view", "services");
+export const currentViewParam = new URLParam<ViewType>("view", "machines");
 export const selectedServiceIdParam = new URLParam("serviceId", "");
 export const selectedMachineIdParam = new URLParam("machineId", ""); 

package/src/diagnostics/errorLogs/ErrorLogController.ts

@@ -9,7 +9,7 @@ import { SocketFunction } from "socket-function/SocketFunction";
 import { requiresNetworkTrustHook } from "../../-d-trust/NetworkTrust2";
 import { ignoreErrors, logErrors, timeoutToError, timeoutToUndefinedSilent } from "../../errors";
 import debugbreak from "debugbreak";
-import { magenta } from "socket-function/src/formatting/logColors";
+import { blue, magenta } from "socket-function/src/formatting/logColors";
 import { formatNumber } from "socket-function/src/formatting/format";
 import { registerShutdownHandler } from "../periodic";
 import { isNoNetwork } from "../../config";
@@ -20,6 +20,7 @@ import { isClient } from "../../config2";
 import { ControllerPick, SocketRegistered } from "socket-function/SocketFunctionTypes";
 
 const NOTIFY_HISTORY = timeInDay * 7;
+const LOG_STORE_HISTORY = timeInDay * 14;
 // Only error / fatal are tracked, for now... we might also add warns?
 const logIssueNotifyTypes: LogType[] = ["error", "fatal"];
 
@@ -55,7 +56,7 @@ export function logSilentWarning(message: string) {
 let nextBlockSeqNum = 1;
 let pendingBlock: LogBlock | undefined;
 export function addLocalLog(log: LogRaw, typeHint: LogType) {
-    // NOTE: Eventually... we should probably ignore logs for non-public machines?
+    // NOTE: Eventually... we should ignore logs for non-public machines?
     //if (isNoNetwork()) return;
     try {
         if (!pendingBlock) {
@@ -84,14 +85,20 @@ export function addLocalLog(log: LogRaw, typeHint: LogType) {
             return;
         }
         let classifier = logClassifier(log, typeHint);
-        pendingBlock.threadId = pendingBlock.threadId || getOwnThreadId();
-        pendingBlock.machineId = pendingBlock.machineId || getOwnMachineId();
+        // Add threadId and machineId, if we have them
+        try {
+            pendingBlock.threadId = pendingBlock.threadId || getOwnThreadId();
+            pendingBlock.machineId = pendingBlock.machineId || getOwnMachineId();
+        } catch { }
         addToLogBlock(pendingBlock, classifier, log);
         ErrorLogControllerBase.onLocalLog(log, classifier);
     } catch (e) {
         internalError(e);
     }
 }
+setTimeout(() => {
+    void initLogClassifier();
+}, 0);
 registerShutdownHandler(flushNow);
 async function flushNow() {
     let newBlock = pendingBlock;
@@ -102,6 +109,19 @@ async function flushNow() {
     if (newBlock.count > 0) {
         let fileInfo = await logBlockToBuffer(newBlock);
         await LOG_ARCHIVES().set(fileInfo.fileName, fileInfo.buffer);
+        if (Math.random() < 0.05) {
+            console.log(blue(`Looking for old log blocks`));
+            let deleteThreshold = Date.now() - LOG_STORE_HISTORY;
+            let files = await LOG_ARCHIVES().find("", { shallow: true, type: "files" });
+            for (let file of files) {
+                if (!file.endsWith(LOG_BLOCK_EXTENSION)) continue;
+                let info = getLogBlockInfo(file);
+                if (info.startTime < deleteThreshold) {
+                    console.log(blue(`Deleting old log block ${file}`));
+                    await LOG_ARCHIVES().del(file);
+                }
+            }
+        }
     }
 }
 
@@ -120,7 +140,6 @@ let ensureBlocksFlushing = lazy(() => {
 let pendingLogs: { log: LogRaw; type: LogType }[] = [];
 let logClassifier: ((log: LogRaw, typeHint: LogType) => LogClass) | undefined;
 let initLogClassifier = lazy(async () => {
-    await timeoutToError(60 * 1000, SocketFunction.mountPromise, () => new Error("Logs lost because service never mounted"));
     let logClasses = await new ErrorLogControllerBase().getClasses();
     logClassifier = getLogClassCategorizer(logClasses);
     if (pendingLogs.length > 0) {

package/src/diagnostics/errorLogs/ErrorLogCore.ts

@@ -14,7 +14,7 @@ export function getMaxExamples(classifier: LogClass): number {
     return classifier.maxExamples ?? MAX_EXAMPLES;
 }
 
-export const LOG_ARCHIVES = lazy(() => getArchives("logs"));
+export const LOG_ARCHIVES = lazy(() => getArchives("notify-error-logs"));
 export const LOG_CLASSES_PATH = "classes.yaml";
 export const LOG_BLOCK_EXTENSION = ".log";
 

package/src/diagnostics/logs/diskShimConsoleLogs.ts

@@ -1,9 +1,19 @@
 import { canHaveChildren } from "socket-function/src/types";
-import { logDisk } from "./diskLogger";
 import { isNode } from "typesafecss";
 import { red, yellow } from "socket-function/src/formatting/logColors";
-import { addLocalLog } from "../errorLogs/ErrorLogController";
-import { LogType } from "../errorLogs/ErrorLogCore";
+
+let addLocalLogModule: (typeof addLocalLogPromise extends Promise<infer T> ? T : never) | undefined;
+const addLocalLogPromise = import("../errorLogs/ErrorLogController").then(x => {
+    (addLocalLogModule as any) = x;
+    return x;
+});
+
+let diskLoggerModule: typeof diskLoggerPromise extends Promise<infer T> ? T : never;
+const diskLoggerPromise = import("./diskLogger").then(x => {
+    (diskLoggerModule as any) = x;
+    return x;
+});
+
 
 let shimmed = false;
 export function shimConsoleLogs() {
@@ -26,10 +36,10 @@ export function shimConsoleLogs() {
                     args.length > 0
                     && String(args[0]).trim().length > 0
                 ) {
-                    if (typeof logDisk === "function") {
+                    if (typeof diskLoggerModule?.logDisk === "function") {
                         // Don't call it directly, so we don't get extra line debug context added to this call
                         //  (as it wouldn't be useful, as we really want the caller)
-                        let stopDoubleShim = logDisk;
+                        let stopDoubleShim = diskLoggerModule.logDisk;
                         stopDoubleShim(...args, { type: fncName });
                     }
                 }
@@ -39,10 +49,10 @@ export function shimConsoleLogs() {
                 // Filter out objects added by injectFileLocationToConsole
                 args = args.filter(x => !(canHaveChildren(x) && x["__FILE__"]));
 
-                if (typeof addLocalLog === "function") {
+                if (typeof addLocalLogModule?.addLocalLog === "function") {
                     if (fncName === "error" || fncName === "warn") {
                         // ALSO, track the logs in a file, for error notifications, etc
-                        addLocalLog({ message: args.join(" | ") + " | " + fileObj?.["__FILE__"], time: Date.now() }, fncName);
+                        addLocalLogModule.addLocalLog({ message: args.join(" | ") + " | " + fileObj?.["__FILE__"], time: Date.now() }, fncName);
                     }
                 }
 

package/src/diagnostics/periodic.ts

@@ -34,8 +34,6 @@ export async function shutdown() {
     if (shuttingDown) {
         return;
     }
-    require("debugbreak")(2);
-    debugger;
     console.log(red("Starting shutdown"));
     shuttingDown = true;
     const { authorityStorage } = await import("../0-path-value-core/pathValueCore");