querysub 0.406.0 → 0.408.0

package/tempnotes.txt

@@ -1,65 +1,12 @@
 
-BUGS:
-    FIXED? The value audit code says that the servers are missing values, And it's trying to send the test values between the different path value servers, even though they should be sharded to never share values. 
-    FIXED? The CYOA server keeps locking up
-        █ PathValueSerializer()                 46.34% (  66.6s = 75.3K *  765us   +  550K * 16.3us )
-    - The values keep syncing to both servers.
-    FIXED? The CYOA and the function server keep failing the audits as well. We might need to look at the logs to see how this could happen. It'll probably be obvious once we look at the logs for those paths.
-        - Is it because we're unwatching it? That would make a lot of sense. 
-        - I guess we can just step in and see if its watched
-            pathWatcher.watchers.get(path)
-            remoteWatcher.remoteWatchPaths.get(path)
-
-
-
-3) gc, to split it into multiple directories
-4) Verify the site still works
-
-5) Verify that the sharded startup only loads in the paths it needs and isn't wasteful
-    - With 100 granularity the waste should be quite low. Unless we get REALLY unlucky? Hmm...
-
-6) Record path waste per PathValueServer, and add an admin warn component that requests this, then shows a warning for PathValueServers that have a high waste load %
-
-
-3) Test sharding again
-4) Verify the test page works well, and shards
-    - We have the function sharding, so the values shouldn't be sent to both servers because the function calls themselves should be sharded. 
-
-
-5) Test full path hash syncing, using data that's NOT in a prefix, and making sure we can sync it with keys to get some of it (and all of it)
-    a) When connecting from a trusted node, everything should work fine, the flag should pass through correctly, as remote watchers should know what flag to pass. 
-    b) When syncing from the client side, only synchronizing all the keys will work. But that does still need to work, and that still requires talking to the correct nodes and passing them the correct flags. 
-
-
-3) add code so that client-side we can get a breakdown of How many paths we have per authority. The goal is to just have most the paths come from a single authority. For reading, it's actually more efficient to talk to a single server rather than many servers, as it's not the amount of data, it's really just the latency overhead of talking to the server. And for writing, same thing, it's a lot easier to just write to one server. And if every single user is reading from every single server, it's just not feasible. It no longer scales well. 
-
-4) Get the AI to go through and add function remapping code. Basically, the idea is that we take the most accessed lookup that we're accessing and we use the key for that as our hash. For simple functions, we could actually automate this, but I'll see how the AI can do. 
-
-5) Verify we are mostly accessing paths on one authority
-
-
-7) Test 4 servers, sharded and redundant as the same time
-8) Verify on the test page
-
-8) Shard FunctionRunner
-    - Now is as good a time as any to deal with any of the sharding issues. And it should work easily enough...
-        - Verify the secondary run backup code works too.
-
-
-2) Add disk verification service
-    - pick a random authority => read the same we would as startup => do an extra backblaze getKeys and getInfo to make sure our cache is accurate => load in the data into memory => verify all old paths with that authority?
-    - I wasn't planning on verifying all paths, but we should be able to. They have to be old enough, of course. But we kind of already have that logic, so the AI can probably write it. The one somewhat hard thing, of course, is doing the hashes, especially because new data could cause the hashes to change because of GCing in memory. But if that's the case, for now we can error. Eventually we will warn and then we will drill down further to find the specific paths that are wrong. And if they actually are wrong, then we can actually error.
-
-
-2) Allow remote servers to talk to our local server, via using the connection the local server makes
-    AH! Add a connectivity page that we can go to, and you can tell all the servers to ping all the other servers, and then it will give us a full list of all that are inaccessible. We should start with this page, deploy it, Then witness that the remote server can't talk to our local servers. They don't even need to be local path value servers, just our local CYOA server, etc. 
-    I think what we need to do is we need to group connections by the resolved identity instead of by the underlying node ID. Maybe this is already done? (But I don't remember doing it).
-
-
 
 3) Deploy EVERYTHING (all services) to remote servers
     3.0) deploy the path value servers shard it like we were testing locally
         - 4, for redundancy and sharding
-    5) Deploy disk verification service
+    3.1) Make sure the gc services are deployed too
+    5) Deploy disk audit service too
+        `yarn audit-disk-values` should work
 
+I think even if we run into some occasional issues, we should just power through and try to fix them later. Because I'm sick of working on the framework...
 
+MONTHLY SUMMARY!

package/test.ts

@@ -2,307 +2,25 @@ import { chdir } from "process";
 chdir("D:/repos/qs-cyoa/");
 
 import "./inject";
-
-import path from "path";
-import { list, sort } from "socket-function/src/misc";
-import { formatNumber } from "socket-function/src/formatting/format";
-import { PathValueArchives, archives, archivesLocks, archivesRecycleBin, pathValueArchives } from "./src/0-path-value-core/pathValueArchives";
-import { trimPathStrToDepth } from "./src/path";
-import { Querysub } from "./src/4-querysub/QuerysubController";
-import { clientWatcher } from "./src/1-path-client/pathValueClientWatcher";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { getControllerNodeIdList } from "./src/-g-core-values/NodeCapabilities";
 import { delay } from "socket-function/src/batching";
-import { authorityStorage } from "./src/0-path-value-core/pathValueCore";
-import { authorityLookup } from "./src/0-path-value-core/AuthorityLookup";
-import { getSnapshot, loadSnapshot, saveSnapshot } from "./src/0-path-value-core/archiveLocks/archiveSnapshots";
-import { Transaction } from "./src/0-path-value-core/archiveLocks/ArchiveLocks2";
-import { getArchives } from "./src/-a-archives/archives";
-import { unique } from "./src/misc";
-import { getAllAuthoritySpec, getOurAuthoritySpec } from "./src/0-path-value-core/PathRouterServerAuthoritySpec";
-import { PathRouter } from "./src/0-path-value-core/PathRouter";
-import { pathValueSerializer } from "./src/-h-path-value-serialize/PathValueSerializer";
-import { getShardPrefixes } from "./src/0-path-value-core/ShardPrefixes";
-
-async function main() {
-    let prefixes = await getShardPrefixes();
-    console.log(prefixes);
-    // let paths = await pathValueArchives.getValuePaths(getAllAuthoritySpec());
-    // paths = paths.filter(x => x.startsWith("P!REMAINING/"));
-    // for (let path of paths) {
-    //     let data = await archives().get(path);
-    //     if (!data) {
-    //         console.log(`Missing data file: ${path}`);
-    //         continue;
-    //     }
-    //     let loadObj = await PathValueArchives.loadValuesFromBuffer({ path, data });
-
-    //     for (let value of loadObj.values) {
-    //         if (!PathRouter.matchesAuthoritySpec(getAllAuthoritySpec(), value.path)) {
-    //             throw new Error(`Value ${value.path} does not match authority spec`);
-    //         }
-    //         console.log(pathValueSerializer.getPathValue(value));
-    //     }
-    // }
-    // return;
+import { green, yellow } from "socket-function/src/formatting/logColors";
+import { Querysub } from "./src/4-querysub/Querysub";
+import { pathValueArchives } from "./src/0-path-value-core/pathValueArchives";
+import { getAllAuthoritySpec } from "./src/0-path-value-core/PathRouterServerAuthoritySpec";
 
-    // let path = "P!REMAINING/data_37c5d323e0f9a083.1ed7a7340a015680.querysubtest.com:52026_1775097754671.579_3_1_337_1775097652829.5793_1775097652829.5793_genesis_.data";
-    // let value = await archives().get(path);
-    // console.log(value?.toString());
-    // return;
 
-    // const path = list(500).map(x => "a").join("");
-    // await archives().set(path, Buffer.from("Hello, world!"));
-    // let value = await archives().get(path);
-    // console.log(value?.toString());
+async function main() {
+    await Querysub.hostService("test");
 
-    // function getName(path: string): string {
-    //     return path.split("/").at(-1)!;
-    // }
+    let spec = await getAllAuthoritySpec();
 
-    //*
-    let allValuesObj = await pathValueArchives.loadValues({
-        nodeId: "",
-        prefixes: [],
-        routeStart: 0,
-        routeEnd: 1,
+    let data = await pathValueArchives.loadValues({
+        ...spec,
+        routeStart: 0.5,
+        routeEnd: 0.6,
     });
-    let allValues = Object.values(allValuesObj.values).flat();
-    console.log(`Total values loaded: ${formatNumber(allValues.length)}`);
-    //*/
-
-    // const badDeleteTransaction = "transaction_   tSeqNum=887   tWriteTime=1774715868374.9143   thread=9dc0   cCount=14   dCount=3482   create=X,X,X,X,X,X,X,X,X,X,X,X,X,X   delete=2@0977,3@0977,4@0977,1@0a2a,2@0a2a,3@0a2a,4@0a2...   .transaction";
-
-
-    // // const snapshotFile = "fileCount=3482   valueCount=1142614   byteCount=1074547019   time=1774724950047.202   hash=9d9acc132ccfbf898ae0c287391e48a88e4e3b3ed3beda16f81a0936e710ab07";
-    // // const snapshotFile2 = "fileCount=51   valueCount=1040817   byteCount=1019830353   time=1772831139982.38   hash=7c6acb6b705a8c68892f787e9c9f958b527bde8b60fb914045863a4bedc269e7";
-    // const fixedSnapshotFile = "fileCount=3482   valueCount=1142614   byteCount=1074547019   time=1774760576390.8218   hash=6ef29678cb600f6920d1615c265ff5150ba97c847143fae2832e26d9e1adf1cf";
-    // let snapshotFileObj = await getSnapshot(fixedSnapshotFile);
-
-    // // let files: string[] = [];
-    // // for (let file of snapshotFileObj.files) {
-    // //     files.push("NEW_remaining/" + getName(file.file));
-    // // }
-    // // let joinedSnapshot = await saveSnapshot({ files });
-    // // console.log(`Joined snapshot: ${joinedSnapshot.file}`);
-    // //await loadSnapshot({ overview: { file: fixedSnapshotFile }, noExit: true, hackNoDelete: true });
-
-    // let snapshot = await getSnapshot(fixedSnapshotFile);
-    // let found = 0;
-    // let recycleBin = 0;
-    // for (let file of snapshot.files) {
-    //     let data = await archives().get(file.file);
-    //     if (data) {
-    //         found++;
-    //         console.log("Found");
-    //     } else if (await archivesRecycleBin().get(file.file)) {
-    //         recycleBin++;
-    //         console.log("Found in recycle bin");
-    //     } else {
-    //         console.log("MISSING!");
-    //     }
-    // }
-    // // console.log(`Found ${found}/${snapshot.files.length} files | Recycle bin: ${recycleBin}`);
-
-    //await loadSnapshot({ overview: { file: fixedSnapshotFile }, noExit: true, hackNoDelete: true });
-
-
-
-    // // let joinedFiles = unique([...snapshot1.files, ...snapshot2.files].map(x => x.file));
-    // // require("debugbreak")(2);
-    // // debugger;
-    // // let joinedSnapshot = await saveSnapshot({ files: joinedFiles });
-    // // console.log(`Joined snapshot: ${joinedSnapshot.file}`);
-    // // let joinedSnapshotFile = "fileCount=3482   valueCount=1142614   byteCount=1074547019   time=1774745293898.8706   hash=9d9acc132ccfbf898ae0c287391e48a88e4e3b3ed3beda16f81a0936e710ab07";
-    // // joinedSnapshotFile = snapshotFile2;
-
-    // // await loadSnapshot({ overview: { file: snapshotFile2 }, noExit: true, hackNoDelete: true });
-
-    /*
-    let allFiles = await getArchives("").find("");
-    let allFileNames = new Set(allFiles.map(getName));
-
-    let joinedSnapshot = await getSnapshot(snapshotFile2);
-
-    let countByExtension = new Map<string, { value: number }>();
-    let found = 0;
-    let recycleBin = 0;
-    for (let file of joinedSnapshot.files) {
-        let extension = file.file.split(".").pop() || "";
-        let existing = countByExtension.get(extension);
-        if (!existing) {
-            existing = { value: 0 };
-            countByExtension.set(extension, existing);
-        }
-        existing.value++;
-        let data = await archives().get(file.file);
-        if (data) {
-            found++;
-        } else if (await archivesRecycleBin().get(file.file)) {
-            recycleBin++;
-        } else if (allFileNames.has(getName(file.file))) {
-            require("debugbreak")(2);
-            debugger;
-        }
-    }
-    console.log(Array.from(countByExtension.entries()).map(([extension, value]) => `${extension}: ${value.value}`).join(" | "));
-    console.log(`Found ${found}/${joinedSnapshot.files.length} files | Recycle bin: ${recycleBin}`);
-
-
-    //*/
-
-
-    // let joined = await saveSnapshot({ files: joinedFiles });
-    // console.log(`Joined snapshot: ${joined.file}`);
-
-    // let recycleBinFiles = await archivesRecycleBin().find("");
-
-    // //let snapshot = await getSnapshot(snapshotFile);
-    //await loadSnapshot({ overview: { file: joinedSnapshotFile }, noExit: true, hackNoDelete: true });
-    // require("debugbreak")(2);
-    // debugger;
-
-    //Ugh... we restored... SOME values. 34.4K. But we should have restored over 1 million. And... it doesn't work, of course.
-
-
-    //let recycleBinFiles = await archivesRecycleBin().find("");
-
-    /*
-    {
-        let snapshot = await getSnapshot(joinedSnapshotFile);
-        let snapshotNames = new Set(snapshot.files.map(x => getName(x.file)));
-
-        let archivesRoot = getArchives("");
-        let allFiles = await archivesRoot.find("");
-        let fullPaths = allFiles.filter(x => snapshotNames.has(getName(x)));
-        let correct = 0;
-        let wrong = 0;
-        let duplicate = 0;
-        let foundPaths = new Set<string>();
-        for (let i = 0; i < fullPaths.length; i++) {
-            let fullPath = fullPaths[i];
-            let name = getName(fullPath);
-            let correctPath = `path-values-recycle-bin/NEW_remaining/${name}`;
-            if (fullPath === correctPath) {
-                correct++;
-                continue;
-            }
-            wrong++;
-            if (foundPaths.has(name)) {
-                duplicate++;
-                continue;
-            }
-            console.log(`Path is wrong at ${i}/${fullPaths.length}: ${fullPath}`);
-            foundPaths.add(name);
-            await archivesRoot.move({
-                path: fullPath,
-                targetPath: correctPath,
-                target: archivesRoot,
-            });
-        }
-        console.log({ correct, wrong, duplicate, snapshotCount: snapshot.files.length, allCount: allFiles.length, fullPathsCount: fullPaths.length });
-    }
-
-    //*/
-
-    // let recycleBinValues = await archivesRecycleBin().findInfo("");
-    // require("debugbreak")(2);
-    // debugger;
-
-    // // File =-.,querysubtest._com.,/data_0ee02faf16c0fadf.1ed7a7340a015680.querysubtest.com:58985_1773251893137_6_1_337_1773251743670_1773251743670_genesis_.data is not in the recycle bin, skipping
-
-    // // UGH... they values we delete are not in the recycle bin? Wtf...
-    // // Welp... files ARE KEPT after deletion natively, so maybe we can find them that way
-
-    // //todonext
-    // // "move" must be broken? Hmm...
-
-    // // HMM... Why do we have all these files in the root? That is surely not correct. These kinda look like our restored snapshot...
-    // // That would make sense if move lost its ability to nest values. Maybe due to our caching?
-
-
-
-    // //let snapshot = await getSnapshot(snapshotFile);
-
-
-
-    // let test = await archives().find("");
-    // console.log(test.length);
-
-
-
-    // We have 125K paths in the recycle bin. We read it surprisingly fast (2 minutes), but yeah... we need to fix this...
-    // So... archivesRecycleBin is VERY slow. Hmm... do we ever clean it up? I think we have to?
-    //  - We will want to thin the snapshots, not just remove all old ones.
-    // Hmm... So what even happened? 
-    //  - We have some transactions. It looks like they just deleted all the data. I guess our GC code accidentally matched everything.
-    // Ah, but... We know the files that were deleted, and they're just moved to the recycle bin. So we should just really go from the transactions to find all of the old values. 
-    // AND... somehow we deleted everything, all transactions, everything.
-    //  - Must have been during the attempt to restore it.
-    // let files = await archivesRecycleBin().findInfo("");
-    // console.log(files);
-    // console.log(files.length);
-    // require("debugbreak")(2);
-    // debugger;
-
-    // let snapshot = await pathValueArchives.loadValues({
-    //     type: "remaining",
-    //     excludedPrefixes: [],
-    // });
-    // let allValues = Object.values(snapshot.values).flat();
-    // console.log(`Total values loaded: ${formatNumber(allValues.length)}`);
-
-
-    // let transactionRaw = await archivesRecycleBin().get(badDeleteTransaction);
-    // if (!transactionRaw) throw new Error(`Transaction not found: ${badDeleteTransaction}`);
-    // console.log(transactionRaw?.toString());
-
-    // let transaction = JSON.parse(transactionRaw.toString()) as Transaction;
-    // let files = transaction.ops.filter(a => a.type === "delete").map(a => a.key);
-    // await saveSnapshot({ files });
-
-
-
-
-    // await Querysub.hostService("test");
-    // //let topology = await authorityLookup.getTopology();
-    // let parentPath = ".,querysubtest._com.,PathFunctionRunner.,book.,Data.,edittingBooks.,hkwnmoiwhuyqenbi.,books.,";
-    // clientWatcher.setWatches({
-    //     paths: new Set([]),
-    //     parentPaths: new Set([parentPath]),
-    //     callback: () => {
-    //         let value = authorityStorage.getPathsFromParent(parentPath);
-    //         console.log({ parentPaths: value?.size, isSynced: authorityStorage.isParentSynced(parentPath) });
-    //     }
-    // });
-
-    // await Querysub.hostService("test");
-
-    // let snapshot = await pathValueArchives.loadValues({
-    //     type: "remaining",
-    //     excludedPrefixes: [],
-    // });
-    // let allValues = Object.values(snapshot.values).flat();
-    // console.log(`Total values loaded: ${formatNumber(allValues.length)}`);
-
-    // let splitLevel = 4;
-    // let paths = new Map<string, {
-    //     count: number;
-    // }>();
-    // for (let value of allValues) {
-    //     let path = value.path;
-    //     let prefix = trimPathStrToDepth(path, splitLevel);
-    //     let entry = paths.get(prefix);
-    //     if (!entry) {
-    //         entry = { count: 0 };
-    //         paths.set(prefix, entry);
-    //     }
-    //     entry.count++;
-    // }
-    // // Log the 10 biggest paths
-    // let entries = Array.from(paths.entries());
-    // sort(entries, x => -x[1].count);
-    // for (let [path, entry] of entries.slice(0, 10)) {
-    //     console.log(`${formatNumber(entry.count)}\t${path}`);
-    // }
 }
 
 main().catch(console.error)

package/src/diagnostics/benchmark.ts

@@ -1,139 +0,0 @@
-import { addToStats, addToStatsValue, createStatsValue, getStatsTop } from "socket-function/src/profiling/stats";
-import type { PathValue } from "../0-path-value-core/pathValueCore";
-import { registerPeriodic } from "./periodic";
-import { formatTime } from "socket-function/src/formatting/format";
-import { formatStats } from "socket-function/src/profiling/statsFormat";
-import { getPathStr1 } from "../path";
-import { registerResource } from "./trackResources";
-import type { CallSpec } from "../3-path-functions/PathFunctionRunner";
-import { PromiseObj } from "../promise";
-import { lazy } from "socket-function/src/caching";
-import { isNode } from "socket-function/src/misc";
-
-const getCallResultPathFnc = lazy(() => import("../4-querysub/querysubPrediction").then((m) => m.getCallResultPath));
-
-// NOTE: There's no point to track non-predicted calls, because if we don't predict them we won't be watching
-//  them, so we won't know when they finish!
-
-class OverlapRateTracker {
-    public openCount = 0;
-    private openTime = 0;
-    public originalOpenTime = Date.now();
-    private openCloseCount = 0;
-
-    public finishedTime = 0;
-    private finishedCount = 0;
-
-    public openValue() {
-        if (this.openCount === 0) {
-            this.openTime = Date.now();
-        }
-        this.openCount++;
-        this.openCloseCount++;
-    }
-    public closeValue() {
-        this.openCount--;
-        let now = Date.now();
-        if (this.openCount === 0 || now - this.openTime > 5000) {
-            this.finishedTime += now - this.openTime;
-            this.openTime = now;
-            this.finishedCount += this.openCloseCount;
-            this.openCloseCount = 0;
-        }
-    }
-    public viewRate() {
-        if (this.finishedTime === 0) return 0;
-        return this.finishedCount / this.finishedTime * 1000;
-    }
-    public saturatedFraction() {
-        return this.finishedTime / (Date.now() - this.originalOpenTime);
-    }
-    public harvestRate() {
-        let rate = this.viewRate();
-        this.finishedTime = 0;
-        this.finishedCount = 0;
-        this.originalOpenTime = Date.now();
-        return rate;
-    }
-}
-
-export class Benchmark {
-
-    public static predictCount = 0;
-    public static receivedCount = 0;
-
-    private static finishedCalls = createStatsValue();
-    public static overlapRate = new OverlapRateTracker();
-
-    private static predictedCalls = registerResource("Benchmark|predictedCalls", new Map<string, number>());
-
-    private static waitingForCalls = registerResource("Benchmark|waitingForCalls", new Map<string, (() => void)[]>());
-
-    public static onStartPredictCall(path: string) {
-        this.predictedCalls.set(path, Date.now());
-        Benchmark.predictCount++;
-        this.overlapRate.openValue();
-    }
-    public static onReceivedValues(values: PathValue[]) {
-        for (let value of values) {
-            if (value.canGCValue) continue;
-            if (!value.valid) continue;
-            let list = this.waitingForCalls.get(value.path);
-            if (!list) continue;
-            this.waitingForCalls.delete(value.path);
-            for (let resolve of list) {
-                resolve();
-            }
-        }
-
-        let now = Date.now();
-        for (let value of values) {
-            if (!value.path.includes(getPathStr1("Results"))) {
-                continue;
-            }
-            if (value.canGCValue) continue;
-            if (!value.valid) continue;
-            let predictedTime = this.predictedCalls.get(value.path);
-            if (!predictedTime) {
-                continue;
-            }
-            this.overlapRate.closeValue();
-            this.predictedCalls.delete(value.path);
-            this.receivedCount++;
-            addToStatsValue(this.finishedCalls, now - predictedTime);;
-        }
-    }
-
-    public static async waitForCallToFinish(call: CallSpec) {
-        let getCallResultPath = await getCallResultPathFnc();
-        let path = getCallResultPath(call);
-        let list = this.waitingForCalls.get(path);
-        if (!list) {
-            list = [];
-            this.waitingForCalls.set(path, list);
-        }
-        let promiseObj = new PromiseObj();
-        list.push(promiseObj.resolve);
-        await promiseObj.promise;
-    }
-
-    public static logCallsNow() {
-        let stats = this.finishedCalls;
-        this.finishedCalls = createStatsValue();
-        console.log(`Calls ${formatStats(stats)}, rate ${this.overlapRate.harvestRate()}`);
-    }
-
-    public static getLiveCallStats() {
-        return this.finishedCalls;
-    }
-    public static getLiveRate() {
-        return this.overlapRate.viewRate();
-    }
-    public static saturatedFraction() {
-        return this.overlapRate.saturatedFraction();
-    }
-}
-
-if (!isNode()) {
-    registerPeriodic(() => Benchmark.logCallsNow());
-}