querysub 0.2.0

package/src/0-path-value-core/archiveLocks/archiveSnapshots.ts

@@ -0,0 +1,180 @@
+import { sha256 } from "js-sha256";
+import { getArchives } from "../../-a-archives/archives";
+import { getAllNodeIds, getOwnThreadId } from "../../-f-node-discovery/NodeDiscovery";
+import { pathValueArchives } from "../pathValueArchives";
+import { ignoreErrors, logErrors, timeoutToUndefinedSilent } from "../../errors";
+import { green, magenta } from "socket-function/src/formatting/logColors";
+
+const snapshots = getArchives("snapshots");
+
+export type ArchiveSnapshotOverview = {
+    // If live, it isn't a real file that can be loaded
+    file: string | "live";
+    hash: string;
+
+    time: number;
+    fileCount: number;
+    valueCount: number;
+    byteCount: number;
+};
+export interface ArchiveSnapshotRead {
+    files: {
+        file: string;
+        state: "live" | "zombie" | "recycled" | "missing";
+        time: number;
+        valueCount: number;
+        byteCount: number;
+        source: string;
+    }[];
+}
+
+function getSnapshotOverview(files: string[]) {
+    let overview: ArchiveSnapshotOverview = {
+        file: "",
+        hash: sha256(files.join("")),
+        time: Date.now(),
+        fileCount: 0,
+        valueCount: 0,
+        byteCount: 0,
+    };
+    for (let file of files) {
+        let obj = pathValueArchives.decodeDataPath(file);
+        overview.fileCount++;
+        overview.valueCount += obj.valueCount;
+        overview.byteCount += obj.byteCount;
+    }
+    overview.file = overviewToFileName(overview);
+    return overview;
+}
+
+export async function saveSnapshot(config: {
+    files: string[],
+}) {
+    let { files } = config;
+    let overview = getSnapshotOverview(files);
+    await snapshots.set(overview.file, Buffer.from(files.join("\n")));
+}
+function overviewToFileName(overview: Omit<ArchiveSnapshotOverview, "file">): string {
+    return Object.entries({
+        fileCount: overview.fileCount,
+        valueCount: overview.valueCount,
+        byteCount: overview.byteCount,
+        time: overview.time,
+        hash: overview.hash,
+    }).map(([key, value]) => `${key}=${value}`).join("   ");
+}
+function fileNameToOverview(fileName: string): ArchiveSnapshotOverview {
+    let parts = fileName.split("   ");
+    let obj = Object.fromEntries(parts.map(part => {
+        let [key, value] = part.split("=");
+        return [key, value];
+    }));
+    return {
+        file: fileName,
+        hash: obj.hash,
+
+        time: +obj.time,
+        fileCount: +obj.fileCount,
+        valueCount: +obj.valueCount,
+        byteCount: +obj.byteCount,
+    };
+}
+
+// pathValueArchives.decodeDataPath
+export async function getSnapshotList(): Promise<ArchiveSnapshotOverview[]> {
+    let snapshotFiles = await snapshots.find("");
+
+    let overview: ArchiveSnapshotOverview[] = [];
+    let locker = await pathValueArchives.getArchiveLocker();
+    let allFiles = await locker.getAllValidFiles();
+
+    let liveOverview = getSnapshotOverview(allFiles.map(x => x.file));
+    liveOverview.file = "live";
+    overview.push(liveOverview);
+
+    overview.push(...snapshotFiles.map(file => fileNameToOverview(file)));
+
+    return overview;
+}
+
+export async function getSnapshot(snapshotFile: string | "live"): Promise<ArchiveSnapshotRead> {
+    let snapshot = await getSnapshotBase(snapshotFile);
+    let locker = await pathValueArchives.getArchiveLocker();
+    let read: ArchiveSnapshotRead = {
+        files: await Promise.all(snapshot.files.map(async file => {
+            let decoded = pathValueArchives.decodeDataPath(file);
+            return {
+                file,
+                state: await locker.unsafeGetFileLocation(file),
+                time: decoded.time,
+                valueCount: decoded.valueCount,
+                byteCount: decoded.byteCount,
+                source: decoded.sourceType,
+            };
+        })),
+    };
+    return read;
+}
+async function getSnapshotBase(snapshotFile: string | "live") {
+    if (snapshotFile === "live") {
+        let locker = await pathValueArchives.getArchiveLocker();
+        let allFiles = await locker.getAllValidFiles();
+        return {
+            time: Date.now(),
+            files: allFiles.map(x => x.file),
+        };
+    }
+    let data = await snapshots.get(snapshotFile);
+    if (!data) {
+        throw new Error(`Snapshot not found: ${snapshotFile}`);
+    }
+    let files = data.toString().split("\n");
+    return {
+        files,
+    };
+}
+
+// EXTREME DANGEROUS! Should only be done when all servers are down (except one, which
+//  is run on a developers machine). Worse case scenario, where everything is broken,
+//  and you need to just go back to a previous state.
+//  - Servers might not be notified, so they will continue to run the wrong data, until
+//      they audit the disk.
+//      - This means new writes SHOULD be rejected, but probably won't, at least on
+//          some servers, which will caused a mixed state.
+//  - Non-atomic, so genesis writes may happen while this is happening. BUT,
+//      you can always just keep applying the snapshot until this doesn't happen.
+//  - DOES save the previous live data as a snapshot, so... is somewhat undoable,
+//      although you will still have to take most of the servers down to get back
+//      to a consistent server state.
+export async function loadSnapshot(config: {
+    overview: ArchiveSnapshotOverview;
+    audit?: (nodeId: string) => Promise<void>;
+}): Promise<void> {
+    let { overview, audit } = config;
+    console.log(magenta(`Loading snapshot: ${overview.file}`));
+    let locker = await pathValueArchives.getArchiveLocker();
+
+    let snapshotData = await snapshots.get(overview.file);
+    if (!snapshotData) {
+        throw new Error(`Snapshot not found: ${overview.file}`);
+    }
+    let files = snapshotData.toString().split("\n");
+
+    console.log(magenta(`Unsafe setting snapshot files: ${overview.file}`));
+    await locker.unsafeSetFiles(files);
+    console.log(magenta(`Finished unsafe setting snapshot files: ${overview.file}`));
+
+    let allNodes = await getAllNodeIds();
+
+    // Logging the errors, not ignoring them
+    if (audit) {
+        await timeoutToUndefinedSilent(10_000, Promise.all(allNodes.map(async nodeId => {
+            if (!audit) return;
+            await audit(nodeId);
+        })));
+    }
+
+    for (let i = 0; i < 10; i++) {
+        console.log(green(`!!!Finished loading snapshot!!!`));
+    }
+}

package/src/0-path-value-core/debugLogs.ts

@@ -0,0 +1,90 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
+import { isDevDebugbreak } from "../config";
+import { measureWrap } from "socket-function/src/profiling/measure";
+
+export interface DebugLog {
+    type: string;
+    time: number;
+    values: { [key: string]: unknown };
+}
+
+const MAX_LOG_HISTORY = 1000 * 1000 * 10;
+let ENABLED_LOGGING = isDevDebugbreak();
+export function enableDebugLogging() {
+    ENABLED_LOGGING = true;
+    debugLogFnc = debugLogBase;
+}
+export function disableDebugLogging() {
+    ENABLED_LOGGING = false;
+    debugLogFnc = NOOP;
+}
+export function isDebugLogEnabled() {
+    return ENABLED_LOGGING;
+}
+
+let logHistory: DebugLog[] = [];
+let nextIndex = 0;
+export function getFullLogHistory() {
+    return logHistory;
+}
+export function getLogHistoryIncludes(includes: string) {
+    return logHistory.filter(x => {
+        return Object.values(x.values).some(y => String(y).includes(includes));
+    });
+}
+export function getLogHistoryEquals(value: string) {
+    return logHistory.filter(x => {
+        return Object.values(x.values).some(y => y === value);
+    });
+}
+
+export function debugLog(type: string, values: { [key: string]: unknown }) {
+    debugLogFnc(type, values);
+}
+let debugLogFnc = debugLogBase;
+let NOOP = () => { };
+function debugLogBase(type: string, values: { [key: string]: unknown }) {
+    if (!ENABLED_LOGGING) {
+        disableDebugLogging();
+        return;
+    }
+    let newEntry: DebugLog = { type, time: Date.now(), values };
+    // NOTE: Use a rolling index. We don't promise our results are sorted, so this is fine.
+    //  ALSO, .shift to make a queue lags (it was making this function profile 90% of the time,
+    //  taking multiple seconds).
+    if (logHistory.length < MAX_LOG_HISTORY) {
+        logHistory.push(newEntry);
+    } else {
+        logHistory[nextIndex] = newEntry;
+        nextIndex = (nextIndex + 1) % MAX_LOG_HISTORY;
+    }
+};
+
+class DebugLogControllerBase {
+    public async getLogHistoryIncludes(includes: string) {
+        return getLogHistoryIncludes(includes);
+    }
+    public async getLogHistoryEquals(value: string) {
+        return getLogHistoryEquals(value);
+    }
+    public async getLogHistorySize() {
+        return {
+            "logHistory.length": logHistory.length,
+        };
+    }
+}
+
+export const DebugLogController = SocketFunction.register(
+    "DebugLogController-169e9f30-2958-48a9-b5f3-06bcaebdb8e0",
+    new DebugLogControllerBase(),
+    () => ({
+        getLogHistoryIncludes: {},
+        getLogHistoryEquals: {},
+        getLogHistorySize: {},
+    }),
+    () => ({
+        hooks: [requiresNetworkTrustHook],
+    })
+);
+