querysub 0.642.0 → 0.644.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.642.0",
+  "version": "0.644.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -79,8 +79,8 @@
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
-    "sliftutils": "^1.7.114",
-    "socket-function": "^1.2.31",
+    "sliftutils": "^1.7.116",
+    "socket-function": "^1.2.33",
     "terser": "^5.31.0",
     "typenode": "^6.6.1",
     "typesafecss": "^0.32.0",

package/src/0-path-value-core/archiveLocks/ArchiveLocks2.ts

@@ -123,11 +123,14 @@ export function createArchiveLocker2(config: {
             // Delete all files we didn't just set (move to recycle bin)
             let moveCount = 0;
             async function moveFile(file: string) {
-                moveCount++;
-                if (correctFiles.has(file)) return;
+                if (correctFiles.has(file)) {
+                    moveCount++;
+                    return;
+                }
                 let info = await archiveValues.getInfo(file);
                 console.log(`Moving file ${file}, ${moveCount}/${valuePaths.size}, ${formatNumber(info?.size || 0)} bytes`);
                 await moveArchiveFile({ from: archiveValues, to: archiveRecycleBin, path: file, noFallbacks: true });
+                moveCount++;
                 console.log(`Moved file ${file}, ${moveCount}/${valuePaths.size}`);
             }
             let moveFileParallel = runInParallel({ parallelCount: 8 }, moveFile);

package/src/0-path-value-core/archiveLocks/archiveSnapshots.ts

@@ -1,6 +1,6 @@
 import { sha256 } from "js-sha256";
 import { getAllNodeIds, getOwnThreadId } from "../../-f-node-discovery/NodeDiscovery";
-import { archives, pathValueArchives } from "../pathValueArchives";
+import { archives, archivesRecycleBin, pathValueArchives } from "../pathValueArchives";
 import { ignoreErrors, logErrors, timeoutToUndefinedSilent } from "../../errors";
 import { green, magenta } from "socket-function/src/formatting/logColors";
 import { devDebugbreak, isPublic } from "../../config";
@@ -111,6 +111,83 @@ export async function getLiveSnapshotList(): Promise<ArchiveSnapshotList> {
     return { loadTime: Date.now() - startTime, snapshots: [liveOverview] };
 }
 
+// Reading every stored snapshot at once is expensive, so we cap the concurrency rather than firing
+//  thousands of reads in parallel.
+const SNAPSHOT_READ_PARALLEL = 32;
+
+export type SnapshotFileCounts = {
+    live: number;
+    recycled: number;
+    missing: number;
+};
+export type RecycleBinCheck = {
+    // Snapshot file name (or "live") => how its files are distributed across live/recycled/missing.
+    counts: { [snapshotFile: string]: SnapshotFileCounts };
+    liveFileCount: number;
+    recycledFileCount: number;
+    snapshotCount: number;
+    loadTime: number;
+};
+
+// Classifies every file of every snapshot as live / recycled / missing, entirely server-side. This
+//  lists the whole live store, the whole recycle bin, and reads every snapshot's file list (all in
+//  parallel), then intersects - so the client gets only the per-snapshot counts, never the (large)
+//  underlying file listings. Expensive (can take minutes), so it's an explicit on-demand action.
+export async function checkRecycleBin(): Promise<RecycleBinCheck> {
+    let startTime = Date.now();
+
+    async function getLiveFilePaths(): Promise<string[]> {
+        // Zero-byte files are deletes on new storage servers, matching how the archive locker filters them.
+        return (await archives().findInfo("")).filter(x => x.size > 0).map(x => x.path);
+    }
+    async function getRecycleBinFilePaths(): Promise<string[]> {
+        return (await archivesRecycleBin().findInfo("")).filter(x => x.size > 0).map(x => x.path);
+    }
+    async function getSnapshotFileLists(): Promise<{ file: string; files: string[] }[]> {
+        let snapshotFiles = await snapshots().find("");
+        let lists: { file: string; files: string[] }[] = [];
+        async function readOne(file: string): Promise<void> {
+            let data = await snapshots().get(file);
+            if (!data) return;
+            lists.push({ file, files: data.toString().split("\n") });
+        }
+        let readParallel = runInParallel({ parallelCount: SNAPSHOT_READ_PARALLEL }, readOne);
+        await Promise.all(snapshotFiles.map(readParallel));
+        return lists;
+    }
+
+    let [liveFiles, recycledFiles, snapshotLists] = await Promise.all([
+        getLiveFilePaths(),
+        getRecycleBinFilePaths(),
+        getSnapshotFileLists(),
+    ]);
+
+    let liveSet = new Set(liveFiles);
+    let recycledSet = new Set(recycledFiles);
+    let counts: { [snapshotFile: string]: SnapshotFileCounts } = {};
+    for (let snapshot of snapshotLists) {
+        let count: SnapshotFileCounts = { live: 0, recycled: 0, missing: 0 };
+        for (let file of snapshot.files) {
+            if (liveSet.has(file)) {
+                count.live++;
+            } else if (recycledSet.has(file)) {
+                count.recycled++;
+            } else {
+                count.missing++;
+            }
+        }
+        counts[snapshot.file] = count;
+    }
+
+    return {
+        counts,
+        liveFileCount: liveFiles.length,
+        recycledFileCount: recycledFiles.length,
+        snapshotCount: snapshotLists.length,
+        loadTime: Date.now() - startTime,
+    };
+}
+
 export async function getSnapshot(snapshotFile: string | "live"): Promise<ArchiveSnapshotRead> {
     let snapshot = await getSnapshotBase(snapshotFile);
     let locker = await pathValueArchives.getArchiveLocker();

package/src/5-diagnostics/GenericFormat.tsx

@@ -173,6 +173,7 @@ export function formatValue(value: unknown, formatter: JSXFormatter = "guess", c
 }
 
 export function toSpaceCase(text: string) {
+    if (typeof text !== "string") return String(text);
     if (text.startsWith("_")) return text;
     return text
         // "camelCase" => "camel Case"

package/src/diagnostics/misc-pages/SnapshotViewer.tsx

@@ -1,7 +1,8 @@
 module.allowclient = true;
 
-import { ArchiveSnapshotList, ArchiveSnapshotOverview, ArchiveSnapshotRead, downloadSnapshot, getLiveSnapshotList, getSavedSnapshotList, getSnapshot, loadSnapshot, saveSnapshot, uploadSnapshot } from "../../0-path-value-core/archiveLocks/archiveSnapshots";
+import { ArchiveSnapshotList, ArchiveSnapshotOverview, ArchiveSnapshotRead, checkRecycleBin as checkRecycleBinServer, downloadSnapshot, getLiveSnapshotList, getSavedSnapshotList, getSnapshot, loadSnapshot, RecycleBinCheck, saveSnapshot, uploadSnapshot } from "../../0-path-value-core/archiveLocks/archiveSnapshots";
 import { qreact } from "../../4-dom/qreact";
+import preact from "preact";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { getBrowserUrlNode } from "../../-f-node-discovery/NodeDiscovery";
 import { Table } from "../../5-diagnostics/Table";
@@ -34,6 +35,16 @@ export class SnapshotViewer extends qreact.Component {
         saving: t.boolean(false),
         uploading: t.boolean(false),
         downloading: t.lookup(t.boolean),
+        // Bulk recycle-bin check. The server classifies every snapshot's files as live/recycled/missing
+        //  and returns just the counts; `active` stays true after it finishes so the columns keep showing.
+        recycleCheck: {
+            active: t.boolean(false),
+            running: t.boolean(false),
+            error: t.optional<string>(),
+            summary: t.optional<string>(),
+            // snapshot file name => classified counts (atomic, so we write the whole triple at once)
+            results: t.lookup(t.atomic<{ live: number; recycled: number; missing: number }>()),
+        },
     });
     render() {
         let controller = SnapshotViewerSynced(getBrowserUrlNode());
@@ -51,6 +62,20 @@ export class SnapshotViewer extends qreact.Component {
         );
         let file = selectedSnapshot.value;
         if (!file || typeof file !== "string") {
+            let recycleCheck = this.state.recycleCheck;
+            let rows: SnapshotListRow[] = snapshotList.map(snapshot => {
+                let counts = recycleCheck.results[snapshot.file];
+                // The live "snapshot" isn't a stored file, but by definition every file in it is live.
+                if (!counts && snapshot.file === "live" && recycleCheck.active) {
+                    counts = { live: snapshot.fileCount, recycled: 0, missing: 0 };
+                }
+                return {
+                    ...snapshot,
+                    liveCount: counts?.live,
+                    recycledCount: counts?.recycled,
+                    missingCount: counts?.missing,
+                };
+            });
             return (
                 <div class={css.pad2(10).vbox(10)}>
                     {loadStatus}
@@ -59,14 +84,23 @@ export class SnapshotViewer extends qreact.Component {
                         <Button onClick={() => triggerSnapshotUpload(this)}>
                             {this.state.uploading && "Uploading..." || "Upload Snapshot"}
                         </Button>
+                        <Button onClick={() => checkRecycleBin(this)}>
+                            {recycleCheck.running && "Checking Recycle Bin..." || "Check Recycle Bin"}
+                        </Button>
                     </div>
+                    {recycleCheck.running && <div>Checking recycle bin (this can take a few minutes)...</div>}
+                    {recycleCheck.summary && <div>{recycleCheck.summary}</div>}
+                    {recycleCheck.error && <div class={css.color("red").whiteSpace("pre-wrap")}>{recycleCheck.error}</div>}
                     <Table
-                        rows={snapshotList}
+                        rows={rows}
                         columns={{
                             time: {},
                             fileCount: {},
                             valueCount: {},
                             byteCount: {},
+                            liveCount: { title: "Live", formatter: formatCountCell },
+                            recycledCount: { title: "Recycled", formatter: formatCountCell },
+                            missingCount: { title: "Missing", formatter: formatMissingCell },
                             file: {
                                 title: "Actions",
                                 formatter: (file, context) => {
@@ -80,7 +114,7 @@ export class SnapshotViewer extends qreact.Component {
                                                 try {
                                                     await SnapshotViewerSynced(getBrowserUrlNode()).saveLiveSnapshot.promise();
                                                 } finally {
-                                                    Querysub.localCommit(() => {
+                                                    Querysub.commit(() => {
                                                         this.state.saving = false;
                                                     });
                                                 }
@@ -202,9 +236,59 @@ function summarizeSnapshotByStatus(files: ArchiveSnapshotRead["files"]): Snapsho
     return rows;
 }
 
+type SnapshotListRow = ArchiveSnapshotOverview & {
+    liveCount?: number;
+    recycledCount?: number;
+    missingCount?: number;
+};
+
+function formatCountCell(value: unknown): string {
+    if (typeof value !== "number") return "";
+    return formatNumber(value);
+}
+function formatMissingCell(value: unknown): preact.ComponentChild {
+    if (typeof value !== "number") return "";
+    if (value === 0) return "0";
+    // Missing files in a snapshot mean it can no longer be fully restored - call that out.
+    return <span class={css.color("red").fontWeight("bold")}>{formatNumber(value)}</span>;
+}
+
+async function checkRecycleBin(component: SnapshotViewer) {
+    if (component.state.recycleCheck.running) return;
+    const controller = SnapshotViewerController.nodes[getBrowserUrlNode()];
+    Querysub.commit(() => {
+        let rc = component.state.recycleCheck;
+        rc.active = true;
+        rc.running = true;
+        rc.error = undefined;
+        rc.summary = undefined;
+        for (let key of Object.keys(rc.results)) {
+            delete rc.results[key];
+        }
+    });
+    try {
+        // The server does all the listing/reading/classifying and returns only the per-snapshot counts.
+        let check = await controller.checkRecycleBin();
+        Querysub.commit(() => {
+            let rc = component.state.recycleCheck;
+            for (let [snapshotFile, counts] of Object.entries(check.counts)) {
+                rc.results[snapshotFile] = counts;
+            }
+            rc.summary = `Checked ${formatNumber(check.snapshotCount)} snapshots against ${formatNumber(check.liveFileCount)} live and ${formatNumber(check.recycledFileCount)} recycled files in ${formatTime(check.loadTime)}.`;
+            rc.running = false;
+        });
+    } catch (e: any) {
+        console.error(red(`Failed to check recycle bin: ${e.stack}`));
+        Querysub.commit(() => {
+            component.state.recycleCheck.error = e.stack || String(e);
+            component.state.recycleCheck.running = false;
+        });
+    }
+}
+
 async function downloadSnapshotFile(component: SnapshotViewer, file: string) {
     if (component.state.downloading[file]) return;
-    Querysub.localCommit(() => {
+    Querysub.commit(() => {
         component.state.downloading[file] = true;
     });
     try {
@@ -224,7 +308,7 @@ async function downloadSnapshotFile(component: SnapshotViewer, file: string) {
     } catch (e: any) {
         console.error(red(`Failed to download snapshot ${file}: ${e.stack}`));
     } finally {
-        Querysub.localCommit(() => {
+        Querysub.commit(() => {
             delete component.state.downloading[file];
         });
     }
@@ -238,7 +322,7 @@ async function triggerSnapshotUpload(component: SnapshotViewer) {
     input.onchange = async () => {
         let selected = input.files?.[0];
         if (!selected) return;
-        Querysub.localCommit(() => {
+        Querysub.commit(() => {
             component.state.uploading = true;
         });
         try {
@@ -249,7 +333,7 @@ async function triggerSnapshotUpload(component: SnapshotViewer) {
         } catch (e: any) {
             console.error(red(`Failed to upload snapshot: ${e.stack}`));
         } finally {
-            Querysub.localCommit(() => {
+            Querysub.commit(() => {
                 component.state.uploading = false;
             });
         }
@@ -316,6 +400,10 @@ class SnapshotViewerControllerBase {
         return await downloadSnapshot(snapshotFile);
     }
 
+    public async checkRecycleBin(): Promise<RecycleBinCheck> {
+        return await checkRecycleBinServer();
+    }
+
     public async uploadSnapshot(buffer: Buffer): Promise<ArchiveSnapshotOverview> {
         return await uploadSnapshot(buffer);
     }
@@ -332,6 +420,8 @@ export const SnapshotViewerController = SocketFunction.register(
         saveLiveSnapshot: {},
         downloadSnapshot: {},
         uploadSnapshot: {},
+        // Returns a per-snapshot count map (not the underlying listings), but there can be many snapshots.
+        checkRecycleBin: { responseLimit: 1024 * 1024 * 1024 },
         example: {}
     }),
     () => ({