querysub 0.405.0 → 0.407.0

package/src/3-path-functions/syncSchema.ts

@@ -21,6 +21,7 @@ import { getSpecFromModule } from "./pathFunctionLoader";
 import { isNode } from "typesafecss";
 import { LOCAL_DOMAIN } from "../0-path-value-core/PathRouter";
 import { createRoutingOverrideKey } from "../0-path-value-core/PathRouterRouteOverride";
+import { delay } from "socket-function/src/batching";
 
 // This is the the function id which should be used when creating the FunctionSpec (in order to load the module),
 //  to access the permissions in the schema.
@@ -286,7 +287,9 @@ type SyncSchemaResult<Schema> = {
 
 let prefixes: string[] = [];
 // ONLY used by the deploy process. Path value server is pretty much the only one who needs this, and they aren't including client-side code, and they definitely aren't updating client-side code, so it doesn't make sense to use this directly.
-export function getPrefixesForDeploy(): string[] {
+export async function getPrefixesForDeploy(): Promise<string[]> {
+    await new Promise(r => setImmediate(r));
+    await new Promise(r => setImmediate(r));
     return prefixes;
 }
 // Used for some old non syncSchema schema cases
@@ -302,7 +305,8 @@ export function syncSchema<Schema>(schema?: Schema2): SyncSchemaResult<Schema> {
         const domainName = config.domainName ?? getDomain();
 
         const data = () => functionSchema()[domainName].PathFunctionRunner[moduleId].Data;
-        let dataCached = lazy(() => data()) as () => Schema;
+        // NOTE: We used to cache this, but then they would cache the current database that's inside of the schema, which is dynamically set. So I don't think caching this really makes sense. We might be able to do some kind of intelligent cache where we check what the current database is and only reuse the cache if that's the same. 
+        let dataCached = (() => data()) as () => Schema;
 
         let fncs = { ...functions };
 

package/src/4-deploy/deployGetFunctionsInner.ts

@@ -94,7 +94,7 @@ async function main() {
         }
     }
 
-    for (let prefix of getPrefixesForDeploy()) {
+    for (let prefix of await getPrefixesForDeploy()) {
         console.log(`PREFIX:${prefix}`);
     }
 }

package/src/4-deploy/deployPrefixes.ts

@@ -0,0 +1,14 @@
+import { setShardPrefixes } from "../0-path-value-core/ShardPrefixes";
+import { getPrefixesForDeploy } from "../3-path-functions/syncSchema";
+import path from "path";
+
+async function main() {
+    let deployPath = path.resolve("./deploy.ts");
+    await import(deployPath);
+
+    let prefixes = await getPrefixesForDeploy();
+    await setShardPrefixes(prefixes);
+}
+
+main().catch(console.error).finally(() => process.exit());
+

package/src/4-deploy/edgeNodes.ts

@@ -273,7 +273,7 @@ async function updateEdgeNodesFile() {
     let liveHash = "";
     if (!noSyncing()) {
         liveHash = await Querysub.commitSynced(() => {
-            return deploySchema()[getDomain()].deploy.live.hash;
+            return String(deploySchema()[getDomain()].deploy.live.hash);
         });
     }
 

package/src/diagnostics/SyncTestPage.tsx

@@ -17,11 +17,9 @@ import { PathAuditerController } from "./pathAuditer";
 import { t } from "../2-proxy/schema2";
 
 
-let { data, functions } = Querysub.createSchema<{
-    values: {
-        [key: string]: number;
-    };
-}>()({
+let { data, functions } = Querysub.createSchema({
+    values: t.lookup(t.number)
+})({
     functions: {
         setValue(key: string, value: number) {
             data().values[key] = value;
@@ -90,6 +88,14 @@ export class SyncTestPage extends qreact.Component {
         allThreads = allThreads.slice();
         sort(allThreads, x => x.entrypoint);
 
+        function refreshDelayed() {
+            Querysub.onCommitFinished(() => {
+                setTimeout(() => {
+                    syncTestController.refreshAll();
+                }, 500);
+            });
+        }
+
         return (
             <div className={css.pad2(20).vbox(40)}>
                 <h1>Sync Test Page</h1>
@@ -125,8 +131,11 @@ export class SyncTestPage extends qreact.Component {
                                 <div className={css.hbox(10)}>
                                     <span class={css.boldStyle}>{key}</span>
                                     <span>{value}</span>
-                                    <Button onClick={() => functions.incrementValue(key)}>Increment</Button>
-                                    <InputLabel value={value} number onChangeValue={value => functions.setValue(key, +value)} />
+                                    <Button onClick={() => { functions.incrementValue(key); refreshDelayed(); }}>Increment</Button>
+                                    <InputLabel value={value} number onChangeValue={value => {
+                                        functions.setValue(key, +value);
+                                        refreshDelayed();
+                                    }} />
                                 </div>
                                 <div className={css.hbox(10, 2).wrap.marginLeft(20)}>
                                     {allThreads.map(thread => {
@@ -149,6 +158,7 @@ export class SyncTestPage extends qreact.Component {
                                                         path: path,
                                                         nodeId: thread.nodeId
                                                     });
+                                                    refreshDelayed();
                                                 }}>
                                                     Delete Value
                                                 </Button>
@@ -168,6 +178,7 @@ export class SyncTestPage extends qreact.Component {
                     label="Insert new key"
                     onChangeValue={value => {
                         functions.setValue(value, 0);
+                        refreshDelayed();
                     }}
                 />
                 <Button onClick={() => {
@@ -205,7 +216,7 @@ class SyncTestControllerBase {
     }
 
     public async getValueAndTime(path: string) {
-        let pathValue = authorityStorage.getValueAtTime(path);
+        let pathValue = authorityStorage.getValueAtTime(path, undefined, false, "noAudit");
         let value = pathValueSerializer.getPathValue(pathValue);
         let time = pathValue?.time.time;
         return { value, time };

package/src/functional/promiseCache.ts

@@ -34,7 +34,10 @@ export function cacheAsyncLimited<Arg, Return>(limit: number, fnc: (arg: Arg) =>
 export const cacheAsyncSynced = cacheAsyncLimitedJSON;
 export function cacheAsyncLimitedJSON<Arg, Return>(
     limit: number,
-    fnc: (arg: Arg) => Promise<Return>
+    fnc: (arg: Arg) => Promise<Return>,
+    config?: {
+        notReadyValue?: Return;
+    }
 ): ((arg: Arg) => Return | undefined) & {
     clear(): void;
     promise: (arg: Arg) => Promise<Return>;
@@ -71,6 +74,6 @@ export function cacheAsyncLimitedJSON<Arg, Return>(
             }
         );
         proxyWatcher.triggerOnPromiseFinish(promise, { waitReason: fnc.toString() });
-        return undefined;
+        return config?.notReadyValue;
     };
 }

package/src/path.ts

@@ -171,9 +171,16 @@ export function getParentPathStr(pathStr: string) {
     return pathStr.slice(0, getStartOfLastPart(pathStr));
 }
 
-/** === getPathFromStr(pathStr).slice(-1)[0] */
+/** === getPathFromStr(pathStr).slice(-1)[0] || "" */
 export function getLastPathPart(pathStr: string) {
-    return unescapePathPart(pathStr.slice(getStartOfLastPart(pathStr) - pathDelimitEscaped.length, -pathDelimitEscaped.length));
+    let lastPartIndex = pathStr.lastIndexOf(pathDelimitEscaped);
+    if (lastPartIndex < 0) return "";
+    return unescapePathPart(
+        pathStr.slice(
+            getStartOfLastPart(pathStr),
+            lastPartIndex,
+        )
+    );
 }
 
 /** === getPathStr(getPathFromStr(pathStr).slice(0, -1)) */
@@ -181,8 +188,12 @@ export function removePathLastPart(pathStr: string) {
     return pathStr.slice(0, getStartOfLastPart(pathStr));
 }
 
+let lastPastPath = { path: "", depth: 0 };
 /** getPathDepth(pathStr) === getPathFromStr(pathStr).length */
 export function getPathDepth(pathStr: string): number {
+    if (lastPastPath.path === pathStr) {
+        return lastPastPath.depth;
+    }
     // Start at 1, to skip the first pathDelimit
     let index = 1;
     // Start at -1, as the last loop doesn't have pathDelimitEscaped, but is still incremented
@@ -191,6 +202,8 @@ export function getPathDepth(pathStr: string): number {
         count++;
         index = pathStr.indexOf(pathDelimitEscaped, index) + 1;
     }
+    lastPastPath.path = pathStr;
+    lastPastPath.depth = count;
     return count;
 }
 

package/src/rangeMath.ts

@@ -0,0 +1,41 @@
+export type Range = {
+    start: number;
+    end: number;
+};
+
+export type Ranges = Range[];
+
+export function rangesOverlap(a: Range, b: Range): boolean {
+    return a.start < b.end && a.end > b.start;
+}
+
+export function removeRange(ranges: Ranges, range: Range): {
+    removedRanges: Range[];
+} {
+    let s = range.start;
+    let e = range.end;
+    let removedRanges: Range[] = [];
+    for (let i = ranges.length - 1; i >= 0; i--) {
+        let rangeInRanges = ranges[i];
+        if (s >= rangeInRanges.end || e <= rangeInRanges.start) continue;
+        let startTaken = Math.max(rangeInRanges.start, s);
+        let endTaken = Math.min(rangeInRanges.end, e);
+        removedRanges.push({ start: startTaken, end: endTaken });
+        ranges.splice(i, 1);
+        // Add back the parts we didn't overlap
+        if (rangeInRanges.start < s) {
+            ranges.push({
+                start: rangeInRanges.start,
+                end: s,
+            });
+        }
+        if (rangeInRanges.end > e) {
+            ranges.push({
+                start: e,
+                end: rangeInRanges.end,
+            });
+        }
+
+    }
+    return { removedRanges };
+}

package/tempnotes.txt

@@ -1,42 +1,24 @@
 
-BUGS:
-    FIXED? The value audit code says that the servers are missing values, And it's trying to send the test values between the different path value servers, even though they should be sharded to never share values. 
-    FIXED? The CYOA server keeps locking up
-        █ PathValueSerializer()                 46.34% (  66.6s = 75.3K *  765us   +  550K * 16.3us )
-    - The values keep syncing to both servers.
-    FIXED? The CYOA and the function server keep failing the audits as well. We might need to look at the logs to see how this could happen. It'll probably be obvious once we look at the logs for those paths.
-        - Is it because we're unwatching it? That would make a lot of sense. 
-        - I guess we can just step in and see if its watched
-            pathWatcher.watchers.get(path)
-            remoteWatcher.remoteWatchPaths.get(path)
+2) Change the test page lookup to be an actual lookup (and make it actually use the type safe schema)
+    - And... deploy, we need to deploy the function too...
 
+3) After making it an actual lookup, it should match with the function sharding. And so this should get rid of the cases when one update goes to both servers (Which happens because without the lookup both the data values and the function values could resolve to be on different servers, meaning both servers will know about the write).
 
-
-3) gc, to split it into multiple directories
-4) Verify the site still works
-
-5) Verify that the sharded startup only loads in the paths it needs and isn't wasteful
-    - With 100 granularity the waste should be quite low. Unless we get REALLY unlucky? Hmm...
-
-6) Record path waste per PathValueServer, and add an admin warn component that requests this, then shows a warning for PathValueServers that have a high waste load %
-
-
-3) Test sharding again
 4) Verify the test page works well, and shards
     - We have the function sharding, so the values shouldn't be sent to both servers because the function calls themselves should be sharded. 
 
 
-5) Test full path hash syncing, using data that's NOT in a prefix, and making sure we can sync it with keys to get some of it (and all of it)
-    a) When connecting from a trusted node, everything should work fine, the flag should pass through correctly, as remote watchers should know what flag to pass. 
-    b) When syncing from the client side, only synchronizing all the keys will work. But that does still need to work, and that still requires talking to the correct nodes and passing them the correct flags. 
-
+3) add code so that client-side we can get a breakdown of How many paths we have per authority. The goal is to have it heavily biased, not balanced. We want to only use a single server. (But then, of course, for different users, we should be using a different server)
+3.1) AND Give a count of how many writes per authority and how many reads, as in how many changes per authority. 
+3.2) Let it popup and show which paths are on each server
 
-3) add code so that client-side we can get a breakdown of How many paths we have per authority. The goal is to just have most the paths come from a single authority. For reading, it's actually more efficient to talk to a single server rather than many servers, as it's not the amount of data, it's really just the latency overhead of talking to the server. And for writing, same thing, it's a lot easier to just write to one server. And if every single user is reading from every single server, it's just not feasible. It no longer scales well. 
 
 4) Get the AI to go through and add function remapping code. Basically, the idea is that we take the most accessed lookup that we're accessing and we use the key for that as our hash. For simple functions, we could actually automate this, but I'll see how the AI can do. 
 
 5) Verify we are mostly accessing paths on one authority
 
+3.3) Do more testing with having a partial shard range being down. Realistically, we should be able to set some kind of range, maybe just a very small range in the middle of the two main servers that isn't covered, and still be able to use the site. Previously, we weren't able to. Maybe this was because the transactions just never resolved because we couldn't synchronize some of the values?
+
 
 7) Test 4 servers, sharded and redundant as the same time
 8) Verify on the test page
@@ -45,6 +27,10 @@ BUGS:
     - Now is as good a time as any to deal with any of the sharding issues. And it should work easily enough...
         - Verify the secondary run backup code works too.
 
+9) Do some tests with adding new modules and new functions in development
+    - I think we need to have the git prefix code. Get some extra prefixes in development.
+        - And likely import the deploy.ts code...
+    - Some kind of "getDevelopmentPrefixes" should be fine, which we dynamically call when is not public in getShardPrefixes
 
 2) Add disk verification service
     - pick a random authority => read the same we would as startup => do an extra backblaze getKeys and getInfo to make sure our cache is accurate => load in the data into memory => verify all old paths with that authority?

package/test.ts

@@ -21,12 +21,16 @@ import { getAllAuthoritySpec, getOurAuthoritySpec } from "./src/0-path-value-cor
 import { PathRouter } from "./src/0-path-value-core/PathRouter";
 import { pathValueSerializer } from "./src/-h-path-value-serialize/PathValueSerializer";
 import { getShardPrefixes } from "./src/0-path-value-core/ShardPrefixes";
+import { getPrefixesForDeploy } from "./src/3-path-functions/syncSchema";
 
 async function main() {
-    let prefixes = await getShardPrefixes();
-    console.log(prefixes);
     let deployPath = path.resolve("./deploy.ts");
     await import(deployPath);
+    let prefixes = await getPrefixesForDeploy();
+    console.log(prefixes);
+    // let prefixes = await getShardPrefixes();
+    // console.log(prefixes);
+    return;
     // let paths = await pathValueArchives.getValuePaths(getAllAuthoritySpec());
     // paths = paths.filter(x => x.startsWith("P!REMAINING/"));
     // for (let path of paths) {
@@ -60,7 +64,7 @@ async function main() {
     //     return path.split("/").at(-1)!;
     // }
 
-    /*
+    //*
     let allValuesObj = await pathValueArchives.loadValues({
         nodeId: "",
         prefixes: [],
@@ -69,7 +73,7 @@ async function main() {
     });
     let allValues = Object.values(allValuesObj.values).flat();
     console.log(`Total values loaded: ${formatNumber(allValues.length)}`);
-    */
+    //*/
 
     // const badDeleteTransaction = "transaction_   tSeqNum=887   tWriteTime=1774715868374.9143   thread=9dc0   cCount=14   dCount=3482   create=X,X,X,X,X,X,X,X,X,X,X,X,X,X   delete=2@0977,3@0977,4@0977,1@0a2a,2@0a2a,3@0a2a,4@0a2...   .transaction";
 

package/src/diagnostics/logs/BrowserLargeFileCache.ts

@@ -1,64 +0,0 @@
-import { SocketFunctionClientHook } from "socket-function/SocketFunctionTypes";
-import { cache, lazy } from "socket-function/src/caching";
-import { decodeCborx, encodeCborx } from "../../misc/cloneHelpers";
-import { sha256 } from "js-sha256";
-import { errorToUndefined } from "../../errors";
-import { isNode } from "typesafecss";
-
-let getRootDirectory = lazy(async () => {
-    await navigator.storage.persist();
-    return navigator.storage.getDirectory();
-});
-
-export const getBrowserLargeFileCache = cache((name: string) => new BrowserLargeFileCache(name));
-
-export class BrowserLargeFileCache {
-    constructor(private name: string) { }
-
-    private getDir = lazy(async () => {
-        let root = await getRootDirectory();
-        return await root.getDirectoryHandle(this.name, { create: true });
-    });
-
-    public async set(key: string, value: Buffer) {
-        let dir = await this.getDir();
-        let file = await dir.getFileHandle(key, { create: true });
-        let writable = await file.createWritable();
-        await writable.write(value);
-        await writable.close();
-    }
-    public async get(key: string): Promise<Buffer | undefined> {
-        let dir = await this.getDir();
-        try {
-            let file = await dir.getFileHandle(key, { create: false });
-            let readable = await file.getFile();
-            return Buffer.from(await readable.arrayBuffer());
-        } catch {
-            return undefined;
-        }
-    }
-}
-
-/** Cache key = [args, functionName, classGuid]
- *      - Not nodeId, as that change so frequently, so caching based on server is not usually useful.
- *          - If you want to cache based on nodeId, pass it as an unused arg.
- * 
- *  Uses a file per key, and never cleans them up. So... basically, this sucks, but, it should
- *      work for a few specific functions with large and consistent values.
- */
-export const cacheCalls: SocketFunctionClientHook = async config => {
-    if (isNode()) return;
-
-    let { args, functionName, classGuid } = config.call;
-    let bucket = sha256(JSON.stringify({ functionName, classGuid }));
-    let cache = getBrowserLargeFileCache(bucket);
-    let key = sha256(encodeCborx(args));
-    let cachedValue = await cache.get(key);
-    if (cachedValue) {
-        config.overrideResult = decodeCborx(cachedValue);
-        return;
-    }
-    config.onResult.push(async (result) => {
-        await errorToUndefined(cache.set(key, encodeCborx(result)));
-    });
-};