querysub 0.411.0 → 0.413.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.411.0",
+  "version": "0.413.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/-c-identity/IdentityController.ts

@@ -147,13 +147,16 @@ class IdentityControllerBase {
         if (!areNodeIdsEqual(payload.serverId, localNodeId)) {
             throw new Error(`Identity is for another server! The connection is calling us ${localNodeId}, but signature is for ${payload.serverId}`);
         }
-        let calledMachineId = getMachineId(payload.serverId);
-        if (calledMachineId !== "127-0-0-1" && calledMachineId !== getOwnMachineId()) {
-            throw new Error(`Tried to call a different machine. We are ${getOwnMachineId()}, they called ${calledMachineId}`);
-        }
-        let calledThreadId = decodeNodeId(payload.serverId)?.threadId;
-        if (calledThreadId && calledThreadId !== "127-0-0-1" && calledThreadId !== getOwnThreadId()) {
-            throw new Error(`Tried to call a different thread. We are ${getOwnThreadId()}, they called ${calledThreadId}`);
+        // If they're calling from the browser, then they're not going to be able to use our machine ID, etc. However, they should be calling an actual https node, so it should still be secure for them. 
+        if (payload.clientIsNode) {
+            let calledMachineId = getMachineId(payload.serverId);
+            if (calledMachineId !== "127-0-0-1" && calledMachineId !== getOwnMachineId()) {
+                throw new Error(`Tried to call a different machine. We are ${getOwnMachineId()}, they called ${calledMachineId}`);
+            }
+            let calledThreadId = decodeNodeId(payload.serverId)?.threadId;
+            if (calledThreadId && calledThreadId !== "127-0-0-1" && calledThreadId !== getOwnThreadId()) {
+                throw new Error(`Tried to call a different thread. We are ${getOwnThreadId()}, they called ${calledThreadId}`);
+            }
         }
 
 

package/src/0-path-value-core/PathValueCommitter.ts

@@ -21,6 +21,8 @@ import { decodeNodeId } from "../-a-auth/certs";
 import { decodeParentFilter, encodeParentFilter } from "./hackedPackedPathParentFiltering";
 import { deepCloneCborx } from "../misc/cloneHelpers";
 import { removeRange } from "../rangeMath";
+import { registerShutdownHandler } from "../diagnostics/periodic";
+import { getCallFactory } from "socket-function/src/nodeCache";
 setImmediate(() => import("../1-path-client/RemoteWatcher"));
 setImmediate(() => import("../4-querysub/Querysub"));
 
@@ -48,6 +50,7 @@ class PathValueCommitter {
         void promise.finally(() => this.pendingCommits.delete(promise));
     }
     public async waitForValuesToCommit() {
+        await this.broadcastValues({ values: new Set() });
         await Promise.all(this.pendingCommits);
         // HACK: Wait a bit more, for the websocket to send the values (and for some batching, etc, etc)
         //await new Promise(resolve => setTimeout(resolve, 100));
@@ -143,6 +146,7 @@ class PathValueCommitter {
             tryCount?: number;
         }[]) {
             let values = new Set(valuesBatched.flatMap(x => Array.from(x.values)));
+            if (values.size === 0) return;
             let tryCountPerValue = new Map<PathValue, number>();
             for (let list of valuesBatched) {
                 for (let value of list.values) {
@@ -216,20 +220,24 @@ class PathValueCommitter {
 
             // Don't send to bad nodes for 60 seconds
             const nodeIgnoreTime = Date.now() - 1000 * 60;
+
             let promises = Array.from(valuesPerOtherAuthority.entries()).map(async ([otherAuthority, values]) => {
 
                 let disconnected = SocketFunction.getLastDisconnectTime(otherAuthority);
-                if (disconnected && disconnected > nodeIgnoreTime) {
+                if (!SocketFunction.isNodeConnected(otherAuthority) && disconnected && disconnected > nodeIgnoreTime) {
                     // If it disconnected recently... don't send to it for a little bit, so we don't spend
                     //  all of our time spamming disconnected nodes
+                    console.log(`disconnected at ${disconnected} > ${nodeIgnoreTime}`);
                     return;
                 }
 
                 let isTrusted = await isTrustedByNode(otherAuthority);
                 if (!isTrusted) {
+                    console.log(`not trusted`);
                     throw new Error(`Tried to write to paths on authorities not trusted by us. You probably need to call a function instead of directly writing to the server schema. Authority: ${otherAuthority}, Paths: ${values.map(x => getPathFromStr(x.path).join(".")).join(", ")}`);
                 }
 
+
                 // NOTE: We don't retry on failure, because... we broadcasted it anyways, AND this is supposed
                 //  to be on a trusted node (which should always be a server), so... either our network went
                 //  down, or all nodes went down. Either way, it likely won't fix itself quickly, and so these
@@ -434,3 +442,7 @@ class PathValueCommitter {
 }
 
 export const pathValueCommitter = new PathValueCommitter();
+
+registerShutdownHandler(async () => {
+    await pathValueCommitter.waitForValuesToCommit();
+});

package/src/0-path-value-core/ValidStateComputer.ts

@@ -27,6 +27,7 @@ class ValidStateComputer {
         parentSyncs: { parentPath: string; sourceNodeId: string }[];
         initialTriggers: { values: Set<string>; parentPaths: Set<string> };
         doNotArchive?: boolean;
+
     }) {
         let { pathValues, parentSyncs, initialTriggers, doNotArchive } = config;
 
@@ -93,7 +94,6 @@ class ValidStateComputer {
         }
 
 
-
         let initialPrevValidStates: Map<PathValue, boolean | undefined> | undefined = this.capturePrevValidStates(ourValues);
         authorityStorage.ingestValues(ourValues, { doNotArchive });
 
@@ -104,6 +104,17 @@ class ValidStateComputer {
         let valuesChanged = new Set<PathValue>(pathValues);
         let dependenciesChanged = new Set<PathValue>(ourValues);
         let alreadyTriggered = new Set<PathValue>(dependenciesChanged);
+        // NOTE: For our own values, we don't care what the input valid state is, compute valid states will calculate if it's changed. 
+        for (let value of notOurValues) {
+            // NOTE: We could be smarter and determine if the value we're receiving has a valid state change. However, it's probably fine. 
+            // NOTE: Any watchers will get their valid states computed. I think that's really the only reason to watch something is if we own it and want to compute its valid state. 
+            let watchers = lockWatcher2.getValuePathWatchers(value);
+            for (let watcher of watchers) {
+                if (alreadyTriggered.has(watcher)) continue;
+                dependenciesChanged.add(watcher);
+                alreadyTriggered.add(watcher);
+            }
+        }
 
         let loopCount = 0;
 

package/src/0-path-value-core/pathValueCore.ts

@@ -880,7 +880,7 @@ class AuthorityPathValueStorage {
 
         let goldenTime = now - MAX_CHANGE_AGE;
         // NOTE: Values with no locks are implicitly golden, as they can't be rejected
-        let firstGoldenIndex = values.findIndex(x => x.time.time < goldenTime && x.valid || x.lockCount === 0);
+        let firstGoldenIndex = values.findIndex(x => x.valid && (x.time.time < goldenTime || x.lockCount === 0));
         if (firstGoldenIndex < 0) return;
 
         // Special case, everything is golden, and the latest value is undefined, and is behind the GC

package/src/4-querysub/querysubPrediction.ts

@@ -260,6 +260,9 @@ function predictCallBase(config: {
             }),
         };
 
+        // Abort, and don't predict
+        if (didCancel) return predictions;
+
         for (let value of predictions.writes) {
             value.source = debugName;
         }
@@ -291,6 +294,7 @@ function predictCallBase(config: {
             pathValues: predictions.writes,
             parentSyncs: [],
             initialTriggers: { values: new Set(), parentPaths: new Set() },
+            doNotArchive: true,
         });
 
         if (Querysub.DEBUG_PREDICTIONS) {
@@ -302,21 +306,24 @@ function predictCallBase(config: {
 
     let didCancel = false;
     function rejectPrediction() {
+        predictions;
         if (didCancel) return;
         didCancel = true;
+        if (!predictions) return;
         // Reject our predictions, as the call likely never got committed, so it will never be written
         validStateComputer.ingestValuesAndValidStates({
-            pathValues: [{
-                path: pathResultWrite,
-                value: undefined,
+            pathValues: predictions.writes.map(write => ({
+                path: write.path,
+                value: write.value,
                 locks: [],
                 lockCount: 0,
                 valid: false,
                 time: predictResultWrite.time,
                 isTransparent: false,
-            }],
+            })),
             parentSyncs: [],
             initialTriggers: { values: new Set(), parentPaths: new Set() },
+            doNotArchive: true,
         });
     }
 

package/src/diagnostics/NodeConnectionsPage.tsx

@@ -56,12 +56,12 @@ export class NodeConnectionsPage extends qreact.Component {
 
         return (
             <div class={css.pad2(10).vbox(20).fillWidth}>
-                <div class={css.vbox(10).fillWidth}>
-                    <h1>Connectable Nodes</h1>
+                <h1>Connectable Nodes</h1>
+                <div class={css.hbox(20).wrap.fillWidth}>
                     {connectableNodes.map(node => {
                         let color = getColorForNodeId(node.friendlyNodeId);
                         return (
-                            <div class={css.vbox(5).pad2(10).maxHeight(600).overflowAuto.bord(1, { h: 0, s: 0, l: 80 })}>
+                            <div class={css.vbox(5).pad2(10).maxHeight(400).overflowAuto.bord(1, { h: 0, s: 0, l: 80 })}>
                                 <div class={css.vbox(3)}>
                                     {node.entryPoint && (
                                         <div class={css.fontSize(14).fontWeight("bold")}>

package/src/diagnostics/logs/errorNotifications2/ErrorWarning.tsx

@@ -3,7 +3,7 @@ import { t } from "../../../2-proxy/schema2";
 import { css } from "typesafecss";
 import { Querysub } from "../../../4-querysub/QuerysubController";
 import { ATag } from "../../../library-components/ATag";
-import { managementPageURL } from "../../managementPages";
+import { managementPageURL, showingManagementURL } from "../../managementPages";
 import { LogDatumRenderer } from "./ErrorNotificationPage";
 import { getErrorNotificationsManager } from "./errorWatcher";
 
@@ -22,7 +22,10 @@ export class ErrorWarning extends qreact.Component {
         let firstError = this.manager.state.unmatchedErrors[0];
 
         return <div className={css.hbox(4).alignItems("start").hsl(0, 0, 90).bord2(0, 0, 85).pad2(4, 2).hslcolor(0, 0, 0)}>
-            <ATag className={css.paddingTop(5).flexShrink0} values={[managementPageURL.getOverride("ErrorNotificationPage")]}>
+            <ATag className={css.paddingTop(5).flexShrink0} values={[
+                showingManagementURL.getOverride(true),
+                managementPageURL.getOverride("ErrorNotificationPage"),
+            ]}>
                 Suppress
             </ATag>
             <div className={css.paddingTop(5)}>|</div>

package/tempnotes.txt

@@ -1,27 +1,12 @@
 
-1) The machines think they're using a local IP, so something's wrong with how they're determining their own IP. 
-2) The machines all think their port is the same, but they shouldn't think that. 
+Alright, well, looks like the writes aren't working. It's not sending it to the server at least. Or at least the server isn't receiving it. 
 
-Oh, we broke our live file at some time. The host of everything is saying it's the same thing. It's just wrong. That's not the host based on the node ID. Where did that even come from? 
+-1) Get a test script that doesn't even use the function runner working. 
 
-Oh, our live hash in the edge node index file is wrong. Let's see if it's wrong in the database or if we're just creating the file wrong. 
-    - As in, it's corrupted, as in it's the string object representation. 
+0) Get our front end working. I guess we need to run the function runner locally, and then if that works, we need to figure out why the remote function runner isn't working. 
+    - Also, see if we run the local query subserver and local function runner, if those writes can be seen remotely. If it's a remote function runner problem but not a remote path value server problem, we should see the writes. 
 
-1) Test local querysub server
-1.0) If it doesn't work, try to run the test script with a simple watcher to see if we can sync paths
-1.1) If it does... run a public querysub server locally?
-
-Well, let's run locally and see if our local query subserver works. 
-
-It's not working. The server isn't starting up. 
-
-
-3) Deploy EVERYTHING (all services) to remote servers
-    3.0) deploy the path value servers shard it like we were testing locally
-        - 4, for redundancy and sharding
-    3.1) Make sure the gc services are deployed too
-    5) Deploy disk audit service too
-        `yarn audit-disk-values` should work
+-1) Fix the remote servers not being able to send values to our local path value server. I don't know why it wouldn't work... They should have changed the name knowing our local name. 
 
 1) Use connection page to verify server can talk to our locally hosted server
 1.1) Verify by breaking into our local server that we are receiving values written on the remote server in the local server. 

package/test.ts

@@ -6,22 +6,55 @@ import { SocketFunction } from "socket-function/SocketFunction";
 import { getControllerNodeIdList } from "./src/-g-core-values/NodeCapabilities";
 import { delay } from "socket-function/src/batching";
 import { green, yellow } from "socket-function/src/formatting/logColors";
-import { Querysub } from "./src/4-querysub/Querysub";
+import { Querysub, t } from "./src/4-querysub/Querysub";
 import { pathValueArchives } from "./src/0-path-value-core/pathValueArchives";
 import { getAllAuthoritySpec } from "./src/0-path-value-core/PathRouterServerAuthoritySpec";
 import { deploySchema } from "./src/4-deploy/deploySchema";
 import { getDomain } from "./src/config";
+import { getProxyPath } from "./src/2-proxy/pathValueProxy";
+import { ClientWatcher } from "./src/1-path-client/pathValueClientWatcher";
+import { RemoteWatcher } from "./src/1-path-client/RemoteWatcher";
+import { PathRouter } from "./src/0-path-value-core/PathRouter";
+import { shutdown } from "./src/diagnostics/periodic";
 
+let tempTestSchema = Querysub.createSchema({
+    value: t.number,
+})({
+    domainName: getDomain(),
+    moduleId: "tempTest",
+    module: module,
+    functions: {},
+});
 
 async function main() {
     await Querysub.hostService("test");
 
-    let test = await Querysub.commitAsync(() => {
-        let live = deploySchema()[getDomain()].deploy.live.hash;
-        console.log({ live });
-        return String(live);
-    });
-    console.log({ test });
+    // let path = getProxyPath(() => tempTestSchema.data().value);
+    // console.log({ path });
+    // let authorities = PathRouter.getAllAuthorities(path);
+    // for (let authority of authorities) {
+    //     console.log({ authority });
+    // }
+
+    // ClientWatcher.DEBUG_READS = true;
+    // ClientWatcher.DEBUG_WRITES = true;
+    // RemoteWatcher.DEBUG = true;
+
+    let value = await Querysub.commitAsync(() => tempTestSchema.data().value);
+    console.log({ value });
+    await Querysub.commitAsync(() => tempTestSchema.data().value++, { doNotStoreWritesAsPredictions: true });
+    await delay(3000);
+    let value2 = await Querysub.commitAsync(() => tempTestSchema.data().value);
+    console.log({ value2 });
+
+    await shutdown();
+
+    // let test = await Querysub.commitAsync(() => {
+    //     let live = deploySchema()[getDomain()].deploy.live.hash;
+    //     console.log({ live });
+    //     return String(live);
+    // });
+    // console.log({ test });
 }
 
 main().catch(console.error)