querysub 0.325.0 → 0.326.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.325.0",
+  "version": "0.326.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/-a-archives/archiveCache.ts

@@ -12,10 +12,11 @@ import { Args } from "socket-function/src/types";
 import { getArchivesBackblaze } from "./archivesBackBlaze";
 import { formatNumber } from "socket-function/src/formatting/format";
 import { SizeLimiter } from "../diagnostics/SizeLimiter";
+import { isPublic } from "../config";
 
 const SIZE_LIMIT = new SizeLimiter({
     diskRoot: getStorageDir(),
-    maxBytes: 1024 * 1024 * 1024 * 50,
+    maxBytes: isPublic() ? 1024 * 1024 * 1024 * 250 : 1024 * 1024 * 1024 * 50,
     // Anything less than this and we can't even load enough weights models for a single task
     minBytes: 1024 * 1024 * 1024 * 8,
     maxDiskFraction: 0.3,

package/src/-c-identity/IdentityController.ts

@@ -191,6 +191,7 @@ const changeIdentityOnce = cacheWeak(async function changeIdentityOnce(connectio
     };
     let signature = sign(threadKeyCert, payload);
     await timeoutToError(
+        // NOTE: This timeout has to be small as if we try to connect to a node to send it something time sensitive such as a PathValue and it takes too long it might result in us having a PathValue which is expired. The threshold is around 60 seconds and so we want to build a timeout calling a few different nodes before the PathValue expires. 
         10 * 1000,
         IdentityController.nodes[nodeId].changeIdentity(signature, payload),
         () => new Error(`Timeout calling changeIdentity for ${nodeId}`)

package/src/-d-trust/NetworkTrust2.ts

@@ -1,7 +1,7 @@
 import { measureWrap } from "socket-function/src/profiling/measure";
 import { getIdentityCA, getMachineId, getOwnMachineId } from "../-a-auth/certs";
 import { getArchives } from "../-a-archives/archives";
-import { isNode, throttleFunction, timeInSecond } from "socket-function/src/misc";
+import { isNode, throttleFunction, timeInHour, timeInSecond } from "socket-function/src/misc";
 import { SocketFunctionHook } from "socket-function/SocketFunctionTypes";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { IdentityController_getMachineId } from "../-c-identity/IdentityController";
@@ -19,6 +19,7 @@ import { magenta } from "socket-function/src/formatting/logColors";
 // Cache the untrust list, to prevent bugs from causing too many backend reads (while also allowing
 //  bad servers which make request before their trust is verified from staying broken).
 const UNTRUST_CACHE_TIME = 30 * timeInSecond;
+const TRUSTED_CACHE_RESET_INTERVAL = timeInHour;
 
 const archives = lazy(() => getArchives("trust2/"));
 
@@ -57,6 +58,8 @@ export const isTrusted = measureWrap(async function isTrusted(machineId: string)
     // See the comment in requiresNetworkTrustHook for why clients have to trust all callers.
     if (isClient()) return true;
 
+    await populateTrustedCache();
+
     if (trustedCache.has(machineId)) {
         return true;
     }
@@ -70,30 +73,11 @@ export const isTrusted = measureWrap(async function isTrusted(machineId: string)
         return false;
     }
 
-    return await isTrustedBase(machineId);
-});
-let trustedCachePopulated = false;
-const isTrustedBase = runInSerial(measureWrap(async function isTrustedBase(machineId: string) {
-    if (!trustedCachePopulated) {
-        trustedCachePopulated = true;
-        let trustedMachineIds = await archives().find("");
-        lastArchivesTrusted = trustedMachineIds.slice();
-        for (let trustedMachineId of trustedMachineIds) {
-            trustedCache.add(trustedMachineId);
-            // NOTE: We don't load trust certs here, as we need to load them on demand in case the trust changes after our initial startup.
-        }
-    } else {
-        // Checking a single entry is a lot faster (as find is slow)
-        let trusted = await archives().get(machineId);
-        if (trusted) {
-            trustedCache.add(machineId);
-        }
+    // Checking a single entry is fast and if we don't trust them they'll be added to untrusted cache so it shouldn't slow things down by too much. 
+    let trusted = await archives().get(machineId);
+    if (trusted) {
+        trustedCache.add(machineId);
     }
-    // Always trust ourself
-    trustedCache.add(getOwnMachineId());
-
-    // NOTE: This only happens to servers that we connect to. Also we only allow the machine ID to be this special ID in the case it's on our domain. And because we use HTTPS when connecting to domains, it means that it must be implicitly trusted if it has a certificate for our domain.
-    trustedCache.add("127-0-0-1");
 
     if (!trustedCache.has(machineId)) {
         untrustedCache.set(machineId, Date.now() + UNTRUST_CACHE_TIME);
@@ -101,7 +85,24 @@ const isTrustedBase = runInSerial(measureWrap(async function isTrustedBase(machi
     } else {
         return true;
     }
-}));
+});
+let populateTrustedCache = lazy(async () => {
+    let trustedMachineIds = await archives().find("");
+    lastArchivesTrusted = trustedMachineIds.slice();
+    for (let trustedMachineId of trustedMachineIds) {
+        trustedCache.add(trustedMachineId);
+    }
+    // Always trust ourself
+    trustedCache.add(getOwnMachineId());
+
+    // NOTE: This only happens to servers that we connect to. Also we only allow the machine ID to be this special ID in the case it's on our domain. And because we use HTTPS when connecting to domains, it means that it must be implicitly trusted if it has a certificate for our domain.
+    trustedCache.add("127-0-0-1");
+
+    setTimeout(() => {
+        trustedCache.clear();
+        populateTrustedCache.reset();
+    }, TRUSTED_CACHE_RESET_INTERVAL);
+});
 
 export async function isNodeTrusted(nodeId: string) {
     let domainName = getNodeIdDomainMaybeUndefined(nodeId);

package/src/diagnostics/logs/FastArchiveViewer.tsx

@@ -571,9 +571,6 @@ export class FastArchiveViewer<T> extends qreact.Component<{
                         flavor="large"
                         fillWidth
                         onKeyUp={this.handleDownload}
-                        ref2={() => {
-                            void this.handleDownload();
-                        }}
                         noEnterKeyBlur
                         placeholder="Filter terms, ex x | y & z"
                     />

package/src/diagnostics/logs/errorNotifications/errorWatchEntry.tsx

@@ -5,22 +5,37 @@ import { timeInSecond } from "socket-function/src/misc";
 import { formatDateTime } from "socket-function/src/formatting/format";
 
 
-//todonext
 
+// 3) Dismissing of certain errors is not working. They keep coming back.
+//      - I think our expiry date comparison code might be wrong. It seems like once they leave the maybe expired range they come back immediately. If we can reproduce this locally, it'll be trivial to debug because the suppression stuff is global, so we can just see if there are any errors, and if there are, we break in on them. 
+// UGH... To debug this, we need to ignore the changes and then we need to work on the other stuff and then we need to come back later and see if those changes have shown up again. We basically need to debug it when it happens. We can't debug it now. It's too late, Now the errors should be showing up because they are expired. 
 
-// 5) Add a channel to watch suppression entries, as once we suppress it, we don't want notifications anymore anywhere, And we don't want to have to wait
-//      - test by logging when we get errors and then having our script which automatically keeps adding errors to keep running so we can verify that we keep logging new errors and then ignore the errors and this suppression should propagate and cause us to no longer omit the errors.
-//      - This will also cause the errors to be suppressed across different HTTP servers. 
 
-// 5) Get IMs actually sending. 
+// The constant error notifications might be fixed now. We'll see tomorrow after all the rolling updates finish. 
+// 4) fix whatever's causing constant error notifications. Something is broadcasting on the Recent Errors Change channel constantly. 
+//      - I guess usually there's no server that's going to be listening on it. So it's... Not that big of a deal, but it's still annoying. 
+
+
+//  4.1) update channel watching so you can specify that you want to watch only on a specific node ID and then update our code so we only watch it on the controller node ID that we're interfacing with. 
+
+
+// 5) Verify our suppression updates broadcast across the channel correctly, causing us to be able to suppress a notification and our watching script to then stop seeing the new updates. Realistically, it's the calling script that stops setting them, but same thing. 
+
+
+
+// 5) Set up the Instant Messaging Sending API. 
+//  - Discord. With beeper it won't really matter what we're messaging. We could also do WhatsApp. It's really all the same.  
 
 // 6) Set up all the code to properly rate limit IMs, batch them, link back to the log page, etc. 
 //      - Just link to the error page for the last week. We don't need to link to anything specific. 
 //      - properly getting the node ID that we're going to be watching, and if it goes down, getting a new one, and ignoring messages from the old node. 
 //      - And if no node exists, we need to warn and then wait. 
 
+
 // 7) Write the digest script, which is very different, but will run in the same entry.
-//      - Separate warnings and errors and also bucket by time bucket 
+//      - Separate warnings and errors and also bucket by time bucket
+//      - suppressed errors by time bucket (but no type, as we definitely don't want to parse all suppressed errors...)
+//      - Time the entire thing, and put that, and the profile, in the digest too! That will give us a good gauge on if the errors/suppressions are getting slow (due to a lot of errors, or a lot of suppression checks!)
 // 8) Write a page that shows the results of the digest in tabs, writing the digest probably just to backblaze 
 //      - For now, just have two tabs, one for errors and one for warnings. 
 //      - If we're going to do a full scan, we might as well show time series data as well. It's trivial. 

package/src/diagnostics/logs/lifeCycleAnalysis/spec.md

@@ -20,16 +20,6 @@ Very small amount of data
     - For now this will just be for:
         - non-suppressed errors
         - suppressed errors
-    - Eventually the goal of this is to add our tracking charts to this. There are some really useful metrics we can track. 
-        - unique visit IPs. 
-        - Percent bounces. 
-        - average visit length and also median visit length top 95" bottom 5" 
-        - average stories read average story percentages read 
-        - percent of first page story views by subscribers percent of fifth page story view by subscribers 
-        - Number of users visiting the fifth page.
-        - New subscriptions
-        - Subscription cancellations
-        - Total subscriptions
 
 
 
@@ -55,51 +45,9 @@ Very small amount of data
         2.1) Collections of life cycles so we can further reduce the complexity. 
     - Uses FastArchiveViewer, but instead of showing a table, shows lifecycles (a derived concept)
         - We save them in backblaze, with a bit of cache for loading them
-    - List of life cycles
-    - Life cycle
-        - Title
-        - Operation list (each supports | / &, but having multiple is even better)
-            - Match filter
-            - Group key extractions (optional, if not set it becomes a singleton)
-                - Just a field name
-                - CAN have multiple, which adds us as multiple life cycles
-                    - With each one being namespaced using the key, so we can tell them apart
-            - Global value setting (optional, if not set it has no global state impact)
-                - A list of set values
-                    - Each one is an expression which can use fields in the object, ex:
-                    - `alivePathValueServers.$threadId = true`
-            - Show AND SHOULDN'T include match filters!
-                - So when we should the count matched, we can show stats for these, which will often be should have "finished", and shouldn't have "error", so we can see completed, and errors
-                - Similar to error notifications, but... it's nice to also have this here, as we could miss the notification, or suppress it, but when we are looking at a life cycle it's relevant skyrockets.
-                - ALSO for start, so we can see cutoff starts!
-    OH! How do we handle cut off starts?
-        - Maybe... we have a "read preload" duration, and... we read that, BUT, only include life cycles which are also in our main selected time. So we don't cut anything off in our main time, but don't add new values which also get cut off!
-    - Same time/machine/thread selector as log viewer
-    - Allow filtering to specific life cycles
-        - After download, shows matches per life cycle
-        - Button to reset to all
-    - Download logs, and scan for selected life cycles
-        - Immediately on load, showing progress and throttling, so it's not too bad
-    - Result 
-        - List of life cycles, with count of each
-        - Table of individual life cycles?
-        - Can then filter within these life cycles by searching
-        - BUT, importantly, if any log is matched in a life cycle, the entire life cycle is matched
-        - AND, global settings for ALL life cycles are applied, not just filtered ones!
-    - Table of result life cycles
-        - Preview shows first matched line
-        - ALSO, shows duration of life cycle!
-        - And start date
-        - Expand to see pin that specific life cycle above
-            - Show list of logs in it, in another table, with searching on each of them
-            - Start date AND duration of each line!
-        - Can pin multiple life cycles (I guess might as well)
-    - Show list of global value expressions as well (limited, but with filtering to search them easily)
-        - Can expand a global value to see object (but again... limited?)
-    - Can select a specific global value path, to have it injected into
-        - a column for before and after each life cycle
-        - A column for after each line within a life cycle
-        - Can also select parent values, to show all paths under that (use ShowMore, to make them manageable...)
+    - show the life cycles and allow viewing just the specific life cycles and then drilling into those life cycles. Also allow viewing multiple at once so we can view a stream that has many life cycles (as in life cycle collections, which we can also save. )
+    - Show overlap when we're showing the list of life cycles by having it first sorted by start time and then have some kind of indicator for how many values after the value it overlaps with. So if it's serial there'll be no overlap. If everything is being queued up and then run in serial we'll see the overlap go from 1, 2, 3, 4, 5, 6, 7, 8, etc. If it's just in parallel, it'll count up to a number and then go up and down a little bit as values are added and removed. If everything's added at once and it's in parallel, then the values will go up, but then they'll very quickly go down.
+        - Maybe we should have some indication for how much parallel overlap there is? Or kind of like how much gap there is to the start time of the next thing and to the end time of the next thing, something like that. 
 
 6) Add life cycles for
     - Node discovery life cycle (new node, check for alive, check for paths, etc)
@@ -112,6 +60,7 @@ Very small amount of data
         - Use different keys for threadId, and, triggeredNodeId, so we can track how a node discovers other nodes, vs how a node is discovered
         - Mark as dead, dead count increases, remove node as dead
             - MAYBE there should be 2 lifecycles, one for all, and one for just creation type stuff (no discovery)
+    - There's a life cycle for something dying which starts when we first see it's dead and ends when the count reaches the maximum. We want to use the capability to change the ending state so we can have it as partially dead but not fully dead or fully dead so we can see if the dead life cycles keep starting but not finishing which would be bad. 
     - Trusted machine lifecycle
         - Check if we need to add trust
         - Add trust to archives
@@ -138,6 +87,10 @@ Very small amount of data
     - Re-enable all of our services as well.
     - Just logs MIGHT be able to do it, but... life cycles should make it a lot easier to correlate logs, which is something we need to do anyways to solve it...
 
+Make sure we check our life cycles for nodes being added and removed to make sure that life cycle path is pretty empty and there aren't nodes constantly being lost and re-added. 
+
+Check the startup lifecycle to make sure we can detect the nodes pretty fast and in parallel, instead of serial
+
 10) Verify old user/fast-log-cache machine folders are deleted
 
 

package/src/library-components/SyncedController.ts

@@ -309,11 +309,19 @@ export function getSyncedController<T extends SocketRegistered>(
                         obj.promise = undefined;
                         obj.invalidated = true;
                         call(...args);
+                        // Assign to itself, to preset the type assumptions typescript makes (otherwise we get an error below)
+                        obj = obj as any;
                         let promise = atomic(obj.promise);
                         if (!promise) {
                             debugger;
                             throw new Error(`Impossible, called function, but promise is not found for ${fncName}`);
                         }
+                        // Don't cache promise calls
+                        void promise.finally(() => {
+                            if (obj.promise === promise) {
+                                obj.promise = undefined;
+                            }
+                        });
                         return promise;
                     });
                 };

package/src/library-components/icons.tsx

@@ -9,6 +9,9 @@ import { renderToString } from "./renderToString";
 import { debugTime } from "../../src/0-path-value-core/pathValueCore";
 import { measureBlock } from "socket-function/src/profiling/measure";
 
+// TODO: Create more icons with: https://www.recraft.ai/project/e2a2200f-bed2-4426-b8f1-e5120b9dc990
+//  - This should also make creating animated SVGs a lot easier. If the base is nice, and we add comments, we can probably get the AI to help animate it
+
 // Most icons are from https://www.figma.com/file/eVpKKmt8uOKmSYKW4LyusF/Free-Icon-Pack-1600%2B-icons-(Community)?node-id=1654-9894&t=0bDbK0bA9KGpswRE-0
 
 // TODO: Add a build step that does this (storing the .svgs in a file), so we don't

package/src/server.ts

@@ -33,7 +33,8 @@ async function main() {
 
     Error.stackTraceLimit = 20;
 
-    //SocketFunction.logMessages = true;
+    // SocketFunction.logMessages = true;
+    // SocketFunction.silent = false;
 
     // ClientWatcher.DEBUG_READS = true;
     // ClientWatcher.DEBUG_WRITES = true;