querysub 0.407.0 → 0.409.0

package/tempnotes.txt

@@ -1,51 +1,16 @@
 
-2) Change the test page lookup to be an actual lookup (and make it actually use the type safe schema)
-    - And... deploy, we need to deploy the function too...
-
-3) After making it an actual lookup, it should match with the function sharding. And so this should get rid of the cases when one update goes to both servers (Which happens because without the lookup both the data values and the function values could resolve to be on different servers, meaning both servers will know about the write).
-
-4) Verify the test page works well, and shards
-    - We have the function sharding, so the values shouldn't be sent to both servers because the function calls themselves should be sharded. 
-
-
-3) add code so that client-side we can get a breakdown of How many paths we have per authority. The goal is to have it heavily biased, not balanced. We want to only use a single server. (But then, of course, for different users, we should be using a different server)
-3.1) AND Give a count of how many writes per authority and how many reads, as in how many changes per authority. 
-3.2) Let it popup and show which paths are on each server
-
-
-4) Get the AI to go through and add function remapping code. Basically, the idea is that we take the most accessed lookup that we're accessing and we use the key for that as our hash. For simple functions, we could actually automate this, but I'll see how the AI can do. 
-
-5) Verify we are mostly accessing paths on one authority
-
-3.3) Do more testing with having a partial shard range being down. Realistically, we should be able to set some kind of range, maybe just a very small range in the middle of the two main servers that isn't covered, and still be able to use the site. Previously, we weren't able to. Maybe this was because the transactions just never resolved because we couldn't synchronize some of the values?
-
-
-7) Test 4 servers, sharded and redundant as the same time
-8) Verify on the test page
-
-8) Shard FunctionRunner
-    - Now is as good a time as any to deal with any of the sharding issues. And it should work easily enough...
-        - Verify the secondary run backup code works too.
-
-9) Do some tests with adding new modules and new functions in development
-    - I think we need to have the git prefix code. Get some extra prefixes in development.
-        - And likely import the deploy.ts code...
-    - Some kind of "getDevelopmentPrefixes" should be fine, which we dynamically call when is not public in getShardPrefixes
-
-2) Add disk verification service
-    - pick a random authority => read the same we would as startup => do an extra backblaze getKeys and getInfo to make sure our cache is accurate => load in the data into memory => verify all old paths with that authority?
-    - I wasn't planning on verifying all paths, but we should be able to. They have to be old enough, of course. But we kind of already have that logic, so the AI can probably write it. The one somewhat hard thing, of course, is doing the hashes, especially because new data could cause the hashes to change because of GCing in memory. But if that's the case, for now we can error. Eventually we will warn and then we will drill down further to find the specific paths that are wrong. And if they actually are wrong, then we can actually error.
-
-
-2) Allow remote servers to talk to our local server, via using the connection the local server makes
-    AH! Add a connectivity page that we can go to, and you can tell all the servers to ping all the other servers, and then it will give us a full list of all that are inaccessible. We should start with this page, deploy it, Then witness that the remote server can't talk to our local servers. They don't even need to be local path value servers, just our local CYOA server, etc. 
-    I think what we need to do is we need to group connections by the resolved identity instead of by the underlying node ID. Maybe this is already done? (But I don't remember doing it).
-
-
 
 3) Deploy EVERYTHING (all services) to remote servers
     3.0) deploy the path value servers shard it like we were testing locally
         - 4, for redundancy and sharding
-    5) Deploy disk verification service
+    3.1) Make sure the gc services are deployed too
+    5) Deploy disk audit service too
+        `yarn audit-disk-values` should work
+
+1) Use connection page to verify server can talk to our locally hosted server
+1.1) Verify by breaking into our local server that we are receiving values written on the remote server in the local server. 
+    - This is extremely important. Without this, we can't run a local server. There's all kinds of issues, but our socket function changes should make this so this just works. 
 
+I think even if we run into some occasional issues, we should just power through and try to fix them later. Because I'm sick of working on the framework...
 
+MONTHLY SUMMARY!

package/test.ts

@@ -2,313 +2,25 @@ import { chdir } from "process";
 chdir("D:/repos/qs-cyoa/");
 
 import "./inject";
-
-import path from "path";
-import { list, sort } from "socket-function/src/misc";
-import { formatNumber } from "socket-function/src/formatting/format";
-import { PathValueArchives, archives, archivesLocks, archivesRecycleBin, pathValueArchives } from "./src/0-path-value-core/pathValueArchives";
-import { trimPathStrToDepth } from "./src/path";
-import { Querysub } from "./src/4-querysub/QuerysubController";
-import { clientWatcher } from "./src/1-path-client/pathValueClientWatcher";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { getControllerNodeIdList } from "./src/-g-core-values/NodeCapabilities";
 import { delay } from "socket-function/src/batching";
-import { authorityStorage } from "./src/0-path-value-core/pathValueCore";
-import { authorityLookup } from "./src/0-path-value-core/AuthorityLookup";
-import { getSnapshot, loadSnapshot, saveSnapshot } from "./src/0-path-value-core/archiveLocks/archiveSnapshots";
-import { Transaction } from "./src/0-path-value-core/archiveLocks/ArchiveLocks2";
-import { getArchives } from "./src/-a-archives/archives";
-import { unique } from "./src/misc";
-import { getAllAuthoritySpec, getOurAuthoritySpec } from "./src/0-path-value-core/PathRouterServerAuthoritySpec";
-import { PathRouter } from "./src/0-path-value-core/PathRouter";
-import { pathValueSerializer } from "./src/-h-path-value-serialize/PathValueSerializer";
-import { getShardPrefixes } from "./src/0-path-value-core/ShardPrefixes";
-import { getPrefixesForDeploy } from "./src/3-path-functions/syncSchema";
-
-async function main() {
-    let deployPath = path.resolve("./deploy.ts");
-    await import(deployPath);
-    let prefixes = await getPrefixesForDeploy();
-    console.log(prefixes);
-    // let prefixes = await getShardPrefixes();
-    // console.log(prefixes);
-    return;
-    // let paths = await pathValueArchives.getValuePaths(getAllAuthoritySpec());
-    // paths = paths.filter(x => x.startsWith("P!REMAINING/"));
-    // for (let path of paths) {
-    //     let data = await archives().get(path);
-    //     if (!data) {
-    //         console.log(`Missing data file: ${path}`);
-    //         continue;
-    //     }
-    //     let loadObj = await PathValueArchives.loadValuesFromBuffer({ path, data });
-
-    //     for (let value of loadObj.values) {
-    //         if (!PathRouter.matchesAuthoritySpec(getAllAuthoritySpec(), value.path)) {
-    //             throw new Error(`Value ${value.path} does not match authority spec`);
-    //         }
-    //         console.log(pathValueSerializer.getPathValue(value));
-    //     }
-    // }
-    // return;
+import { green, yellow } from "socket-function/src/formatting/logColors";
+import { Querysub } from "./src/4-querysub/Querysub";
+import { pathValueArchives } from "./src/0-path-value-core/pathValueArchives";
+import { getAllAuthoritySpec } from "./src/0-path-value-core/PathRouterServerAuthoritySpec";
 
-    // let path = "P!REMAINING/data_37c5d323e0f9a083.1ed7a7340a015680.querysubtest.com:52026_1775097754671.579_3_1_337_1775097652829.5793_1775097652829.5793_genesis_.data";
-    // let value = await archives().get(path);
-    // console.log(value?.toString());
-    // return;
 
-    // const path = list(500).map(x => "a").join("");
-    // await archives().set(path, Buffer.from("Hello, world!"));
-    // let value = await archives().get(path);
-    // console.log(value?.toString());
+async function main() {
+    await Querysub.hostService("test");
 
-    // function getName(path: string): string {
-    //     return path.split("/").at(-1)!;
-    // }
+    let spec = await getAllAuthoritySpec();
 
-    //*
-    let allValuesObj = await pathValueArchives.loadValues({
-        nodeId: "",
-        prefixes: [],
-        routeStart: 0,
-        routeEnd: 1,
+    let data = await pathValueArchives.loadValues({
+        ...spec,
+        routeStart: 0.5,
+        routeEnd: 0.6,
     });
-    let allValues = Object.values(allValuesObj.values).flat();
-    console.log(`Total values loaded: ${formatNumber(allValues.length)}`);
-    //*/
-
-    // const badDeleteTransaction = "transaction_   tSeqNum=887   tWriteTime=1774715868374.9143   thread=9dc0   cCount=14   dCount=3482   create=X,X,X,X,X,X,X,X,X,X,X,X,X,X   delete=2@0977,3@0977,4@0977,1@0a2a,2@0a2a,3@0a2a,4@0a2...   .transaction";
-
-
-    // // const snapshotFile = "fileCount=3482   valueCount=1142614   byteCount=1074547019   time=1774724950047.202   hash=9d9acc132ccfbf898ae0c287391e48a88e4e3b3ed3beda16f81a0936e710ab07";
-    // // const snapshotFile2 = "fileCount=51   valueCount=1040817   byteCount=1019830353   time=1772831139982.38   hash=7c6acb6b705a8c68892f787e9c9f958b527bde8b60fb914045863a4bedc269e7";
-    // const fixedSnapshotFile = "fileCount=3482   valueCount=1142614   byteCount=1074547019   time=1774760576390.8218   hash=6ef29678cb600f6920d1615c265ff5150ba97c847143fae2832e26d9e1adf1cf";
-    // let snapshotFileObj = await getSnapshot(fixedSnapshotFile);
-
-    // // let files: string[] = [];
-    // // for (let file of snapshotFileObj.files) {
-    // //     files.push("NEW_remaining/" + getName(file.file));
-    // // }
-    // // let joinedSnapshot = await saveSnapshot({ files });
-    // // console.log(`Joined snapshot: ${joinedSnapshot.file}`);
-    // //await loadSnapshot({ overview: { file: fixedSnapshotFile }, noExit: true, hackNoDelete: true });
-
-    // let snapshot = await getSnapshot(fixedSnapshotFile);
-    // let found = 0;
-    // let recycleBin = 0;
-    // for (let file of snapshot.files) {
-    //     let data = await archives().get(file.file);
-    //     if (data) {
-    //         found++;
-    //         console.log("Found");
-    //     } else if (await archivesRecycleBin().get(file.file)) {
-    //         recycleBin++;
-    //         console.log("Found in recycle bin");
-    //     } else {
-    //         console.log("MISSING!");
-    //     }
-    // }
-    // // console.log(`Found ${found}/${snapshot.files.length} files | Recycle bin: ${recycleBin}`);
-
-    //await loadSnapshot({ overview: { file: fixedSnapshotFile }, noExit: true, hackNoDelete: true });
-
-
-
-    // // let joinedFiles = unique([...snapshot1.files, ...snapshot2.files].map(x => x.file));
-    // // require("debugbreak")(2);
-    // // debugger;
-    // // let joinedSnapshot = await saveSnapshot({ files: joinedFiles });
-    // // console.log(`Joined snapshot: ${joinedSnapshot.file}`);
-    // // let joinedSnapshotFile = "fileCount=3482   valueCount=1142614   byteCount=1074547019   time=1774745293898.8706   hash=9d9acc132ccfbf898ae0c287391e48a88e4e3b3ed3beda16f81a0936e710ab07";
-    // // joinedSnapshotFile = snapshotFile2;
-
-    // // await loadSnapshot({ overview: { file: snapshotFile2 }, noExit: true, hackNoDelete: true });
-
-    /*
-    let allFiles = await getArchives("").find("");
-    let allFileNames = new Set(allFiles.map(getName));
-
-    let joinedSnapshot = await getSnapshot(snapshotFile2);
-
-    let countByExtension = new Map<string, { value: number }>();
-    let found = 0;
-    let recycleBin = 0;
-    for (let file of joinedSnapshot.files) {
-        let extension = file.file.split(".").pop() || "";
-        let existing = countByExtension.get(extension);
-        if (!existing) {
-            existing = { value: 0 };
-            countByExtension.set(extension, existing);
-        }
-        existing.value++;
-        let data = await archives().get(file.file);
-        if (data) {
-            found++;
-        } else if (await archivesRecycleBin().get(file.file)) {
-            recycleBin++;
-        } else if (allFileNames.has(getName(file.file))) {
-            require("debugbreak")(2);
-            debugger;
-        }
-    }
-    console.log(Array.from(countByExtension.entries()).map(([extension, value]) => `${extension}: ${value.value}`).join(" | "));
-    console.log(`Found ${found}/${joinedSnapshot.files.length} files | Recycle bin: ${recycleBin}`);
-
-
-    //*/
-
-
-    // let joined = await saveSnapshot({ files: joinedFiles });
-    // console.log(`Joined snapshot: ${joined.file}`);
-
-    // let recycleBinFiles = await archivesRecycleBin().find("");
-
-    // //let snapshot = await getSnapshot(snapshotFile);
-    //await loadSnapshot({ overview: { file: joinedSnapshotFile }, noExit: true, hackNoDelete: true });
-    // require("debugbreak")(2);
-    // debugger;
-
-    //Ugh... we restored... SOME values. 34.4K. But we should have restored over 1 million. And... it doesn't work, of course.
-
-
-    //let recycleBinFiles = await archivesRecycleBin().find("");
-
-    /*
-    {
-        let snapshot = await getSnapshot(joinedSnapshotFile);
-        let snapshotNames = new Set(snapshot.files.map(x => getName(x.file)));
-
-        let archivesRoot = getArchives("");
-        let allFiles = await archivesRoot.find("");
-        let fullPaths = allFiles.filter(x => snapshotNames.has(getName(x)));
-        let correct = 0;
-        let wrong = 0;
-        let duplicate = 0;
-        let foundPaths = new Set<string>();
-        for (let i = 0; i < fullPaths.length; i++) {
-            let fullPath = fullPaths[i];
-            let name = getName(fullPath);
-            let correctPath = `path-values-recycle-bin/NEW_remaining/${name}`;
-            if (fullPath === correctPath) {
-                correct++;
-                continue;
-            }
-            wrong++;
-            if (foundPaths.has(name)) {
-                duplicate++;
-                continue;
-            }
-            console.log(`Path is wrong at ${i}/${fullPaths.length}: ${fullPath}`);
-            foundPaths.add(name);
-            await archivesRoot.move({
-                path: fullPath,
-                targetPath: correctPath,
-                target: archivesRoot,
-            });
-        }
-        console.log({ correct, wrong, duplicate, snapshotCount: snapshot.files.length, allCount: allFiles.length, fullPathsCount: fullPaths.length });
-    }
-
-    //*/
-
-    // let recycleBinValues = await archivesRecycleBin().findInfo("");
-    // require("debugbreak")(2);
-    // debugger;
-
-    // // File =-.,querysubtest._com.,/data_0ee02faf16c0fadf.1ed7a7340a015680.querysubtest.com:58985_1773251893137_6_1_337_1773251743670_1773251743670_genesis_.data is not in the recycle bin, skipping
-
-    // // UGH... they values we delete are not in the recycle bin? Wtf...
-    // // Welp... files ARE KEPT after deletion natively, so maybe we can find them that way
-
-    // //todonext
-    // // "move" must be broken? Hmm...
-
-    // // HMM... Why do we have all these files in the root? That is surely not correct. These kinda look like our restored snapshot...
-    // // That would make sense if move lost its ability to nest values. Maybe due to our caching?
-
-
-
-    // //let snapshot = await getSnapshot(snapshotFile);
-
-
-
-    // let test = await archives().find("");
-    // console.log(test.length);
-
-
-
-    // We have 125K paths in the recycle bin. We read it surprisingly fast (2 minutes), but yeah... we need to fix this...
-    // So... archivesRecycleBin is VERY slow. Hmm... do we ever clean it up? I think we have to?
-    //  - We will want to thin the snapshots, not just remove all old ones.
-    // Hmm... So what even happened? 
-    //  - We have some transactions. It looks like they just deleted all the data. I guess our GC code accidentally matched everything.
-    // Ah, but... We know the files that were deleted, and they're just moved to the recycle bin. So we should just really go from the transactions to find all of the old values. 
-    // AND... somehow we deleted everything, all transactions, everything.
-    //  - Must have been during the attempt to restore it.
-    // let files = await archivesRecycleBin().findInfo("");
-    // console.log(files);
-    // console.log(files.length);
-    // require("debugbreak")(2);
-    // debugger;
-
-    // let snapshot = await pathValueArchives.loadValues({
-    //     type: "remaining",
-    //     excludedPrefixes: [],
-    // });
-    // let allValues = Object.values(snapshot.values).flat();
-    // console.log(`Total values loaded: ${formatNumber(allValues.length)}`);
-
-
-    // let transactionRaw = await archivesRecycleBin().get(badDeleteTransaction);
-    // if (!transactionRaw) throw new Error(`Transaction not found: ${badDeleteTransaction}`);
-    // console.log(transactionRaw?.toString());
-
-    // let transaction = JSON.parse(transactionRaw.toString()) as Transaction;
-    // let files = transaction.ops.filter(a => a.type === "delete").map(a => a.key);
-    // await saveSnapshot({ files });
-
-
-
-
-    // await Querysub.hostService("test");
-    // //let topology = await authorityLookup.getTopology();
-    // let parentPath = ".,querysubtest._com.,PathFunctionRunner.,book.,Data.,edittingBooks.,hkwnmoiwhuyqenbi.,books.,";
-    // clientWatcher.setWatches({
-    //     paths: new Set([]),
-    //     parentPaths: new Set([parentPath]),
-    //     callback: () => {
-    //         let value = authorityStorage.getPathsFromParent(parentPath);
-    //         console.log({ parentPaths: value?.size, isSynced: authorityStorage.isParentSynced(parentPath) });
-    //     }
-    // });
-
-    // await Querysub.hostService("test");
-
-    // let snapshot = await pathValueArchives.loadValues({
-    //     type: "remaining",
-    //     excludedPrefixes: [],
-    // });
-    // let allValues = Object.values(snapshot.values).flat();
-    // console.log(`Total values loaded: ${formatNumber(allValues.length)}`);
-
-    // let splitLevel = 4;
-    // let paths = new Map<string, {
-    //     count: number;
-    // }>();
-    // for (let value of allValues) {
-    //     let path = value.path;
-    //     let prefix = trimPathStrToDepth(path, splitLevel);
-    //     let entry = paths.get(prefix);
-    //     if (!entry) {
-    //         entry = { count: 0 };
-    //         paths.set(prefix, entry);
-    //     }
-    //     entry.count++;
-    // }
-    // // Log the 10 biggest paths
-    // let entries = Array.from(paths.entries());
-    // sort(entries, x => -x[1].count);
-    // for (let [path, entry] of entries.slice(0, 10)) {
-    //     console.log(`${formatNumber(entry.count)}\t${path}`);
-    // }
 }
 
 main().catch(console.error)

package/src/diagnostics/benchmark.ts

@@ -1,139 +0,0 @@
-import { addToStats, addToStatsValue, createStatsValue, getStatsTop } from "socket-function/src/profiling/stats";
-import type { PathValue } from "../0-path-value-core/pathValueCore";
-import { registerPeriodic } from "./periodic";
-import { formatTime } from "socket-function/src/formatting/format";
-import { formatStats } from "socket-function/src/profiling/statsFormat";
-import { getPathStr1 } from "../path";
-import { registerResource } from "./trackResources";
-import type { CallSpec } from "../3-path-functions/PathFunctionRunner";
-import { PromiseObj } from "../promise";
-import { lazy } from "socket-function/src/caching";
-import { isNode } from "socket-function/src/misc";
-
-const getCallResultPathFnc = lazy(() => import("../4-querysub/querysubPrediction").then((m) => m.getCallResultPath));
-
-// NOTE: There's no point to track non-predicted calls, because if we don't predict them we won't be watching
-//  them, so we won't know when they finish!
-
-class OverlapRateTracker {
-    public openCount = 0;
-    private openTime = 0;
-    public originalOpenTime = Date.now();
-    private openCloseCount = 0;
-
-    public finishedTime = 0;
-    private finishedCount = 0;
-
-    public openValue() {
-        if (this.openCount === 0) {
-            this.openTime = Date.now();
-        }
-        this.openCount++;
-        this.openCloseCount++;
-    }
-    public closeValue() {
-        this.openCount--;
-        let now = Date.now();
-        if (this.openCount === 0 || now - this.openTime > 5000) {
-            this.finishedTime += now - this.openTime;
-            this.openTime = now;
-            this.finishedCount += this.openCloseCount;
-            this.openCloseCount = 0;
-        }
-    }
-    public viewRate() {
-        if (this.finishedTime === 0) return 0;
-        return this.finishedCount / this.finishedTime * 1000;
-    }
-    public saturatedFraction() {
-        return this.finishedTime / (Date.now() - this.originalOpenTime);
-    }
-    public harvestRate() {
-        let rate = this.viewRate();
-        this.finishedTime = 0;
-        this.finishedCount = 0;
-        this.originalOpenTime = Date.now();
-        return rate;
-    }
-}
-
-export class Benchmark {
-
-    public static predictCount = 0;
-    public static receivedCount = 0;
-
-    private static finishedCalls = createStatsValue();
-    public static overlapRate = new OverlapRateTracker();
-
-    private static predictedCalls = registerResource("Benchmark|predictedCalls", new Map<string, number>());
-
-    private static waitingForCalls = registerResource("Benchmark|waitingForCalls", new Map<string, (() => void)[]>());
-
-    public static onStartPredictCall(path: string) {
-        this.predictedCalls.set(path, Date.now());
-        Benchmark.predictCount++;
-        this.overlapRate.openValue();
-    }
-    public static onReceivedValues(values: PathValue[]) {
-        for (let value of values) {
-            if (value.canGCValue) continue;
-            if (!value.valid) continue;
-            let list = this.waitingForCalls.get(value.path);
-            if (!list) continue;
-            this.waitingForCalls.delete(value.path);
-            for (let resolve of list) {
-                resolve();
-            }
-        }
-
-        let now = Date.now();
-        for (let value of values) {
-            if (!value.path.includes(getPathStr1("Results"))) {
-                continue;
-            }
-            if (value.canGCValue) continue;
-            if (!value.valid) continue;
-            let predictedTime = this.predictedCalls.get(value.path);
-            if (!predictedTime) {
-                continue;
-            }
-            this.overlapRate.closeValue();
-            this.predictedCalls.delete(value.path);
-            this.receivedCount++;
-            addToStatsValue(this.finishedCalls, now - predictedTime);;
-        }
-    }
-
-    public static async waitForCallToFinish(call: CallSpec) {
-        let getCallResultPath = await getCallResultPathFnc();
-        let path = getCallResultPath(call);
-        let list = this.waitingForCalls.get(path);
-        if (!list) {
-            list = [];
-            this.waitingForCalls.set(path, list);
-        }
-        let promiseObj = new PromiseObj();
-        list.push(promiseObj.resolve);
-        await promiseObj.promise;
-    }
-
-    public static logCallsNow() {
-        let stats = this.finishedCalls;
-        this.finishedCalls = createStatsValue();
-        console.log(`Calls ${formatStats(stats)}, rate ${this.overlapRate.harvestRate()}`);
-    }
-
-    public static getLiveCallStats() {
-        return this.finishedCalls;
-    }
-    public static getLiveRate() {
-        return this.overlapRate.viewRate();
-    }
-    public static saturatedFraction() {
-        return this.overlapRate.saturatedFraction();
-    }
-}
-
-if (!isNode()) {
-    registerPeriodic(() => Benchmark.logCallsNow());
-}