querysub 0.412.0 → 0.414.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.412.0",
+  "version": "0.414.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/-c-identity/IdentityController.ts

@@ -19,6 +19,7 @@ import { waitForFirstTimeSync } from "socket-function/time/trueTimeShim";
 import { red } from "socket-function/src/formatting/logColors";
 import { isNode } from "typesafecss";
 import { areNodeIdsEqual, getOwnNodeId, getOwnThreadId } from "../-f-node-discovery/NodeDiscovery";
+import { timeInMinute } from "socket-function/src/misc";
 
 let callerInfo = new Map<CallerContext, {
     reconnectNodeId: string | undefined;
@@ -235,8 +236,8 @@ const changeIdentityOnce = cacheWeak(async function changeIdentityOnce(connectio
     };
     let signature = sign(threadKeyCert, payload);
     await timeoutToError(
-        // NOTE: This timeout has to be small as if we try to connect to a node to send it something time sensitive such as a PathValue and it takes too long it might result in us having a PathValue which is expired. The threshold is around 60 seconds and so we want to build a timeout calling a few different nodes before the PathValue expires. 
-        10 * 1000,
+        // NOTE: This used to be small, but we cache this, so it would mean a node on startup would time out, and then we would refuse to talk to it ever again. So... this can't be small
+        timeInMinute * 3,
         IdentityController.nodes[nodeId].changeIdentity(signature, payload),
         () => new Error(`Timeout calling changeIdentity for ${nodeId}`)
     );

package/src/0-path-value-core/PathValueCommitter.ts

@@ -21,6 +21,8 @@ import { decodeNodeId } from "../-a-auth/certs";
 import { decodeParentFilter, encodeParentFilter } from "./hackedPackedPathParentFiltering";
 import { deepCloneCborx } from "../misc/cloneHelpers";
 import { removeRange } from "../rangeMath";
+import { registerShutdownHandler } from "../diagnostics/periodic";
+import { getCallFactory } from "socket-function/src/nodeCache";
 setImmediate(() => import("../1-path-client/RemoteWatcher"));
 setImmediate(() => import("../4-querysub/Querysub"));
 
@@ -48,6 +50,7 @@ class PathValueCommitter {
         void promise.finally(() => this.pendingCommits.delete(promise));
     }
     public async waitForValuesToCommit() {
+        await this.broadcastValues({ values: new Set() });
         await Promise.all(this.pendingCommits);
         // HACK: Wait a bit more, for the websocket to send the values (and for some batching, etc, etc)
         //await new Promise(resolve => setTimeout(resolve, 100));
@@ -143,6 +146,7 @@ class PathValueCommitter {
             tryCount?: number;
         }[]) {
             let values = new Set(valuesBatched.flatMap(x => Array.from(x.values)));
+            if (values.size === 0) return;
             let tryCountPerValue = new Map<PathValue, number>();
             for (let list of valuesBatched) {
                 for (let value of list.values) {
@@ -216,20 +220,24 @@ class PathValueCommitter {
 
             // Don't send to bad nodes for 60 seconds
             const nodeIgnoreTime = Date.now() - 1000 * 60;
+
             let promises = Array.from(valuesPerOtherAuthority.entries()).map(async ([otherAuthority, values]) => {
 
                 let disconnected = SocketFunction.getLastDisconnectTime(otherAuthority);
-                if (disconnected && disconnected > nodeIgnoreTime) {
+                if (!SocketFunction.isNodeConnected(otherAuthority) && disconnected && disconnected > nodeIgnoreTime) {
                     // If it disconnected recently... don't send to it for a little bit, so we don't spend
                     //  all of our time spamming disconnected nodes
+                    console.log(`disconnected at ${disconnected} > ${nodeIgnoreTime}`);
                     return;
                 }
 
                 let isTrusted = await isTrustedByNode(otherAuthority);
                 if (!isTrusted) {
+                    console.log(`not trusted`);
                     throw new Error(`Tried to write to paths on authorities not trusted by us. You probably need to call a function instead of directly writing to the server schema. Authority: ${otherAuthority}, Paths: ${values.map(x => getPathFromStr(x.path).join(".")).join(", ")}`);
                 }
 
+
                 // NOTE: We don't retry on failure, because... we broadcasted it anyways, AND this is supposed
                 //  to be on a trusted node (which should always be a server), so... either our network went
                 //  down, or all nodes went down. Either way, it likely won't fix itself quickly, and so these
@@ -434,3 +442,7 @@ class PathValueCommitter {
 }
 
 export const pathValueCommitter = new PathValueCommitter();
+
+registerShutdownHandler(async () => {
+    await pathValueCommitter.waitForValuesToCommit();
+});

package/src/0-path-value-core/ValidStateComputer.ts

@@ -27,6 +27,7 @@ class ValidStateComputer {
         parentSyncs: { parentPath: string; sourceNodeId: string }[];
         initialTriggers: { values: Set<string>; parentPaths: Set<string> };
         doNotArchive?: boolean;
+
     }) {
         let { pathValues, parentSyncs, initialTriggers, doNotArchive } = config;
 
@@ -93,7 +94,6 @@ class ValidStateComputer {
         }
 
 
-
         let initialPrevValidStates: Map<PathValue, boolean | undefined> | undefined = this.capturePrevValidStates(ourValues);
         authorityStorage.ingestValues(ourValues, { doNotArchive });
 
@@ -104,6 +104,17 @@ class ValidStateComputer {
         let valuesChanged = new Set<PathValue>(pathValues);
         let dependenciesChanged = new Set<PathValue>(ourValues);
         let alreadyTriggered = new Set<PathValue>(dependenciesChanged);
+        // NOTE: For our own values, we don't care what the input valid state is, compute valid states will calculate if it's changed. 
+        for (let value of notOurValues) {
+            // NOTE: We could be smarter and determine if the value we're receiving has a valid state change. However, it's probably fine. 
+            // NOTE: Any watchers will get their valid states computed. I think that's really the only reason to watch something is if we own it and want to compute its valid state. 
+            let watchers = lockWatcher2.getValuePathWatchers(value);
+            for (let watcher of watchers) {
+                if (alreadyTriggered.has(watcher)) continue;
+                dependenciesChanged.add(watcher);
+                alreadyTriggered.add(watcher);
+            }
+        }
 
         let loopCount = 0;
 

package/src/0-path-value-core/pathValueCore.ts

@@ -880,7 +880,7 @@ class AuthorityPathValueStorage {
 
         let goldenTime = now - MAX_CHANGE_AGE;
         // NOTE: Values with no locks are implicitly golden, as they can't be rejected
-        let firstGoldenIndex = values.findIndex(x => x.time.time < goldenTime && x.valid || x.lockCount === 0);
+        let firstGoldenIndex = values.findIndex(x => x.valid && (x.time.time < goldenTime || x.lockCount === 0));
         if (firstGoldenIndex < 0) return;
 
         // Special case, everything is golden, and the latest value is undefined, and is behind the GC

package/src/4-querysub/querysubPrediction.ts

@@ -260,6 +260,9 @@ function predictCallBase(config: {
             }),
         };
 
+        // Abort, and don't predict
+        if (didCancel) return predictions;
+
         for (let value of predictions.writes) {
             value.source = debugName;
         }
@@ -291,6 +294,7 @@ function predictCallBase(config: {
             pathValues: predictions.writes,
             parentSyncs: [],
             initialTriggers: { values: new Set(), parentPaths: new Set() },
+            doNotArchive: true,
         });
 
         if (Querysub.DEBUG_PREDICTIONS) {
@@ -302,21 +306,24 @@ function predictCallBase(config: {
 
     let didCancel = false;
     function rejectPrediction() {
+        predictions;
         if (didCancel) return;
         didCancel = true;
+        if (!predictions) return;
         // Reject our predictions, as the call likely never got committed, so it will never be written
         validStateComputer.ingestValuesAndValidStates({
-            pathValues: [{
-                path: pathResultWrite,
-                value: undefined,
+            pathValues: predictions.writes.map(write => ({
+                path: write.path,
+                value: write.value,
                 locks: [],
                 lockCount: 0,
                 valid: false,
                 time: predictResultWrite.time,
                 isTransparent: false,
-            }],
+            })),
             parentSyncs: [],
             initialTriggers: { values: new Set(), parentPaths: new Set() },
+            doNotArchive: true,
         });
     }
 

package/src/diagnostics/NodeConnectionsPage.tsx

@@ -56,12 +56,12 @@ export class NodeConnectionsPage extends qreact.Component {
 
         return (
             <div class={css.pad2(10).vbox(20).fillWidth}>
-                <div class={css.vbox(10).fillWidth}>
-                    <h1>Connectable Nodes</h1>
+                <h1>Connectable Nodes</h1>
+                <div class={css.hbox(20).wrap.fillWidth}>
                     {connectableNodes.map(node => {
                         let color = getColorForNodeId(node.friendlyNodeId);
                         return (
-                            <div class={css.vbox(5).pad2(10).maxHeight(600).overflowAuto.bord(1, { h: 0, s: 0, l: 80 })}>
+                            <div class={css.vbox(5).pad2(10).maxHeight(400).overflowAuto.bord(1, { h: 0, s: 0, l: 80 })}>
                                 <div class={css.vbox(3)}>
                                     {node.entryPoint && (
                                         <div class={css.fontSize(14).fontWeight("bold")}>

package/src/diagnostics/logs/errorNotifications2/ErrorWarning.tsx

@@ -3,7 +3,7 @@ import { t } from "../../../2-proxy/schema2";
 import { css } from "typesafecss";
 import { Querysub } from "../../../4-querysub/QuerysubController";
 import { ATag } from "../../../library-components/ATag";
-import { managementPageURL } from "../../managementPages";
+import { managementPageURL, showingManagementURL } from "../../managementPages";
 import { LogDatumRenderer } from "./ErrorNotificationPage";
 import { getErrorNotificationsManager } from "./errorWatcher";
 
@@ -22,7 +22,10 @@ export class ErrorWarning extends qreact.Component {
         let firstError = this.manager.state.unmatchedErrors[0];
 
         return <div className={css.hbox(4).alignItems("start").hsl(0, 0, 90).bord2(0, 0, 85).pad2(4, 2).hslcolor(0, 0, 0)}>
-            <ATag className={css.paddingTop(5).flexShrink0} values={[managementPageURL.getOverride("ErrorNotificationPage")]}>
+            <ATag className={css.paddingTop(5).flexShrink0} values={[
+                showingManagementURL.getOverride(true),
+                managementPageURL.getOverride("ErrorNotificationPage"),
+            ]}>
                 Suppress
             </ATag>
             <div className={css.paddingTop(5)}>|</div>

package/tempnotes.txt

@@ -1,32 +1,56 @@
 
-1) The machines think they're using a local IP, so something's wrong with how they're determining their own IP. 
-2) The machines all think their port is the same, but they shouldn't think that. 
 
-Oh, we broke our live file at some time. The host of everything is saying it's the same thing. It's just wrong. That's not the host based on the node ID. Where did that even come from? 
+Local CYOA + Local FunctionRunner works
 
-Oh, our live hash in the edge node index file is wrong. Let's see if it's wrong in the database or if we're just creating the file wrong. 
-    - As in, it's corrupted, as in it's the string object representation. 
 
-1) Test local querysub server
-1.0) If it doesn't work, try to run the test script with a simple watcher to see if we can sync paths
-1.1) If it does... run a public querysub server locally?
 
-Well, let's run locally and see if our local query subserver works. 
 
-It's not working. The server isn't starting up. 
+Hmm... we ask to watch it, but the watch doesn't show up on the other side? Fuck...
 
 
-3) Deploy EVERYTHING (all services) to remote servers
-    3.0) deploy the path value servers shard it like we were testing locally
-        - 4, for redundancy and sharding
-    3.1) Make sure the gc services are deployed too
-    5) Deploy disk audit service too
-        `yarn audit-disk-values` should work
+
+
+So we have the remote watcher, so it looks like we were watching it, but we don't think it's synced, so we clearly didn't get a value back. 
+
+WATCHER FAILED TO SYNC findFunctionsToCall DID NOT RECEIVE PATH VALUES. This means PathValueServer is not responding to watches, either to specific paths, or for all paths [
+  '.,querysubtest._com.,',
+  '.,querysubtest._com.,PathFunctionRunner.,'
+] [ '.,querysubtest._com.,PathFunctionRunner.,' ] [Function: findFunctionsToCall]
+Node list is missing nodes, resyncing node {
+  missingNodes: [ '716181d6570f7b55.a794fbcf7b104c68.querysubtest.com:34689' ],
+
+
+UGH... poking fixed it. WHICH MEANS, we have to break it again, and get into the bad state again!
+    - Ugh...
+
+-1) We got into a bad state where The query sub nodes weren't really listening or it's it th they had zombie watchers that hadn't finished synchronizing, but it wasn't complaining about being able to un to find any any paths. There were no unwatched paths 
+    - It fixed when we poked them. 
+
+0) If it fails after redeploying, disable the server and run everything locally again, as we made a lot of changes, and so it might not even work locally. 
+
+-1) Get a test script that doesn't even use the function runner working. 
+
+0) Get our front end working. I guess we need to run the function runner locally, and then if that works, we need to figure out why the remote function runner isn't working. 
+    - Also, see if we run the local query subserver and local function runner, if those writes can be seen remotely. If it's a remote function runner problem but not a remote path value server problem, we should see the writes. 
+
+-1) Fix the remote servers not being able to send values to our local path value server. I don't know why it wouldn't work... They should have changed the name knowing our local name. 
 
 1) Use connection page to verify server can talk to our locally hosted server
 1.1) Verify by breaking into our local server that we are receiving values written on the remote server in the local server. 
     - This is extremely important. Without this, we can't run a local server. There's all kinds of issues, but our socket function changes should make this so this just works. 
 
+0) SHARD THE FUNCTION RUNNER!
+    - And secondary sharding for backup...
+    - First shard locally
+    - Test on our sync test page (Otherwise all the writes are going to go to the same function runner anyway, so it's not a good test. 
+
+-2) Errors are not being automatically grouped. 
+    - We are getting notifications though, which is weird. The notifications are supposed to be automatically grouping it. We should probably look at its logs and see why. 
+
+0) We just are constantly "Node list is missing nodes, resyncing node"
+    - We should check the logs. I'm assuming what happened is that at some point some server thought it was the wrong node ID and deleted it?
+    - It's generally the same ones, except new ones get added. 
+
 I think even if we run into some occasional issues, we should just power through and try to fix them later. Because I'm sick of working on the framework...
 
 MONTHLY SUMMARY!

package/test.ts

@@ -6,22 +6,55 @@ import { SocketFunction } from "socket-function/SocketFunction";
 import { getControllerNodeIdList } from "./src/-g-core-values/NodeCapabilities";
 import { delay } from "socket-function/src/batching";
 import { green, yellow } from "socket-function/src/formatting/logColors";
-import { Querysub } from "./src/4-querysub/Querysub";
+import { Querysub, t } from "./src/4-querysub/Querysub";
 import { pathValueArchives } from "./src/0-path-value-core/pathValueArchives";
 import { getAllAuthoritySpec } from "./src/0-path-value-core/PathRouterServerAuthoritySpec";
 import { deploySchema } from "./src/4-deploy/deploySchema";
 import { getDomain } from "./src/config";
+import { getProxyPath } from "./src/2-proxy/pathValueProxy";
+import { ClientWatcher } from "./src/1-path-client/pathValueClientWatcher";
+import { RemoteWatcher } from "./src/1-path-client/RemoteWatcher";
+import { PathRouter } from "./src/0-path-value-core/PathRouter";
+import { shutdown } from "./src/diagnostics/periodic";
 
+let tempTestSchema = Querysub.createSchema({
+    value: t.number,
+})({
+    domainName: getDomain(),
+    moduleId: "tempTest",
+    module: module,
+    functions: {},
+});
 
 async function main() {
     await Querysub.hostService("test");
 
-    let test = await Querysub.commitAsync(() => {
-        let live = deploySchema()[getDomain()].deploy.live.hash;
-        console.log({ live });
-        return String(live);
-    });
-    console.log({ test });
+    // let path = getProxyPath(() => tempTestSchema.data().value);
+    // console.log({ path });
+    // let authorities = PathRouter.getAllAuthorities(path);
+    // for (let authority of authorities) {
+    //     console.log({ authority });
+    // }
+
+    // ClientWatcher.DEBUG_READS = true;
+    // ClientWatcher.DEBUG_WRITES = true;
+    // RemoteWatcher.DEBUG = true;
+
+    let value = await Querysub.commitAsync(() => tempTestSchema.data().value);
+    console.log({ value });
+    await Querysub.commitAsync(() => tempTestSchema.data().value++, { doNotStoreWritesAsPredictions: true });
+    await delay(3000);
+    let value2 = await Querysub.commitAsync(() => tempTestSchema.data().value);
+    console.log({ value2 });
+
+    await shutdown();
+
+    // let test = await Querysub.commitAsync(() => {
+    //     let live = deploySchema()[getDomain()].deploy.live.hash;
+    //     console.log({ live });
+    //     return String(live);
+    // });
+    // console.log({ test });
 }
 
 main().catch(console.error)