querysub 0.424.0 → 0.425.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.424.0",
+  "version": "0.425.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/0-path-value-core/PathValueCommitter.ts

@@ -15,7 +15,7 @@ import debugbreak from "debugbreak";
 import { red } from "socket-function/src/formatting/logColors";
 import { isClient } from "../config2";
 import { auditLog, isDebugLogEnabled } from "./auditLogs";
-import { authorityLookup } from "./AuthorityLookup";
+import { AuthorityEntry, authorityLookup } from "./AuthorityLookup";
 import { debugNodeId } from "../-c-identity/IdentityController";
 import { decodeNodeId } from "../-a-auth/certs";
 import { decodeParentFilter, encodeParentFilter } from "./hackedPackedPathParentFiltering";
@@ -28,6 +28,10 @@ setImmediate(() => import("../4-querysub/Querysub"));
 
 const MAX_SEND_TRY_COUNT = 3;
 
+// NOTE: This isn't very efficient, but it is safer. It's quadratic depending on the number of authorities for the server. Which shouldn't be so bad. I mean, why would we have four times redundancy? That's a lot. And even then, we're sending sixteen times the amount of traffic. That's fine, I guess. I think it's more reasonable for us to have two times redundancy, maybe even three times redundancy. With two times redundancy, it's only sending the value four times, which is reasonable. And the trade-off is there's basically no way that we can lose the data.
+//      - If we only send to one authority and then they rebroadcast it, it's possible that they die after receiving the value. If we send it to both value servers but they don't re-broadcast, it's possible that we die while broadcasting the values after having only sent a few. However, by sending it to both servers and having both servers send it to the other servers, it means that if we die while broadcasting it, it's fine. And if the server dies that we want to send it to, it's fine because we send it to both servers.
+const BROADCAST_TO_ALL_AUTHORITIES = true;
+
 export type BatchValues = {
     pathValues: PathValue[],
     parentsSynced?: string[];
@@ -207,8 +211,9 @@ class PathValueCommitter {
                         source: pathValue.source,
                         otherAuthorities,
                     });
+                    continue;
                 }
-                for (let otherAuthority of otherAuthorities) {
+                function sendToAuthority(otherAuthority: AuthorityEntry) {
                     let values = valuesPerOtherAuthority.get(otherAuthority.nodeId);
                     if (!values) {
                         values = [];
@@ -216,6 +221,15 @@ class PathValueCommitter {
                     }
                     values.push(pathValue);
                 }
+                if (BROADCAST_TO_ALL_AUTHORITIES) {
+                    for (let otherAuthority of otherAuthorities) {
+                        sendToAuthority(otherAuthority);
+                    }
+                } else {
+                    // NOTE: Path routing is pretty strict. Once we disconnect, it'll remove it from the list of authorities. And if we can't talk to it, the retry logic will call us again.
+                    let otherAuthority = otherAuthorities[~~(Math.random() * otherAuthorities.length)];
+                    sendToAuthority(otherAuthority);
+                }
             }
 
             // Don't send to bad nodes for 60 seconds
@@ -268,30 +282,41 @@ class PathValueCommitter {
                         });
                     }
                 });
-                void forwardPromise.catch(async error => {
-                    let byTryCount = new Map<number, PathValue[]>();
-                    for (let value of values) {
-                        let tryCount = tryCountPerValue.get(value) ?? 0;
-                        let arr = byTryCount.get(tryCount) ?? [];
-                        arr.push(value);
-                        byTryCount.set(tryCount, arr);
-                    }
-                    for (let [tryCount, values] of byTryCount.entries()) {
-                        tryCount++;
-                        if (tryCount > MAX_SEND_TRY_COUNT) {
-                            console.error(`Failed to send values after ${MAX_SEND_TRY_COUNT} tries. Giving up.`, {
-                                error: error.message,
-                                otherAuthority,
-                                count: values.length,
+                // If we broadcast to all authorities, we can't also retry, as this means if one authority is down, we end up infinitely looping. And it breaks the servers and the logs. 
+                if (!BROADCAST_TO_ALL_AUTHORITIES) {
+                    void forwardPromise.catch(async error => {
+                        let byTryCount = new Map<number, PathValue[]>();
+                        for (let value of values) {
+                            let tryCount = tryCountPerValue.get(value) ?? 0;
+                            let arr = byTryCount.get(tryCount) ?? [];
+                            arr.push(value);
+                            byTryCount.set(tryCount, arr);
+                        }
+                        for (let [tryCount, values] of byTryCount.entries()) {
+                            tryCount++;
+                            if (tryCount > MAX_SEND_TRY_COUNT) {
+                                console.error(`Failed to send values after ${MAX_SEND_TRY_COUNT} tries. Giving up.`, {
+                                    error: error.message,
+                                    otherAuthority,
+                                    count: values.length,
+                                });
+                                continue;
+                            }
+                            for (let value of values) {
+                                console.error(`Retrying to send values after ${tryCount} tries.`, {
+                                    error: error.message,
+                                    otherAuthority,
+                                    path: value.path,
+                                    timeId: value.time.time,
+                                });
+                            }
+                            void pathValueCommitter.broadcastValues({
+                                values: new Set(values),
+                                tryCount: tryCount,
                             });
-                            continue;
                         }
-                        void pathValueCommitter.broadcastValues({
-                            values: new Set(values),
-                            tryCount: tryCount,
-                        });
-                    }
-                });
+                    });
+                }
 
                 pathValueCommitter.addCommitPromise(forwardPromise);
             });
@@ -352,9 +377,10 @@ class PathValueCommitter {
                             return true;
                         });
 
-                        for (let value of Array.from(batch.initialTriggers.values)) {
-                            if (isWrongAuthority(value, undefined, "initialTrigger")) {
-                                batch.initialTriggers.values.delete(value);
+                        // NOTE: If we are the authority on it, it means we're not going to have a remote watch node ID. And if we did, it would be ourselves. So this also implicitly filters out any initial triggers for values that we are the authority on. 
+                        for (let path of Array.from(batch.initialTriggers.values)) {
+                            if (isWrongAuthority(path, undefined, "initialTrigger")) {
+                                batch.initialTriggers.values.delete(path);
                             }
                         }
                         for (let parentPath of Array.from(batch.initialTriggers.parentPaths)) {
@@ -362,11 +388,6 @@ class PathValueCommitter {
                                 continue;
                             }
 
-                            // TODO: Remove this breakpoint eventually. This can happen naturally when servers go down?
-                            require("debugbreak")(2);
-                            debugger;
-                            remoteWatcher.isFinalRemoteWatchPath({ parentPath, nodeId: batch.sourceNodeId });
-
                             console.warn(`Ignoring parent path which we aren't watching. From ${debugNodeId(batch.sourceNodeId)}.`, {
                                 parentPath,
                                 sourceNodeId: debugNodeId(batch.sourceNodeId),

package/src/0-path-value-core/PathValueController.ts

@@ -310,8 +310,9 @@ export class PathValueControllerBase {
     public async getValuesByPathAndTime(entries: AuditSnapshotEntry[]): Promise<Buffer[]> {
         let values: PathValue[] = [];
         for (let entry of entries) {
-            let value = authorityStorage.getValueExactIgnoreInvalid(entry.path, entry.time);
+            let value = authorityStorage.getValueExact(entry.path, entry.time);
             if (!value) continue;
+            if (!value.valid) continue;
             if (value.isTransparent) continue;
             if (value.canGCValue) continue;
             values.push(value);

package/src/0-path-value-core/ValidStateComputer.ts

@@ -2,7 +2,7 @@ import { keyByArray, binarySearchIndex } from "socket-function/src/misc";
 import { measureFnc } from "socket-function/src/profiling/measure";
 import { isNode } from "typesafecss";
 import { isDiskAudit } from "../config";
-import { getPathFromStr } from "../path";
+import { getParentPathStr, getPathFromStr } from "../path";
 import { auditLog } from "./auditLogs";
 import { PathRouter } from "./PathRouter";
 import { PathValue, authorityStorage, compareTime, debugPathValuePath, ReadLock, byLockGroup, isCoreQuiet, debugRejections, debugTime, debugPathValue, MAX_CHANGE_AGE } from "./pathValueCore";
@@ -40,13 +40,42 @@ class ValidStateComputer {
 
         let ourValues: PathValue[] = [];
         let notOurValues: PathValue[] = [];
-        for (let value of pathValues) {
+        pathValues = pathValues.filter(value => {
+            // NOTE: I think if it's initial trigger we need to process it anyways?
+            if (!config.initialTriggers.values.has(value.path)) {
+                // Ignore values we already have. We receive duplicates, often due to values being broadcast multiple times in order to prevent data loss.
+                let existingValue = authorityStorage.getValueExact(value.path, value.time);
+                // If we already have the value and the valid state is the exact same, then remove it from path values and don't even add it to our values. Pretend like we never received it.
+                if (existingValue && !!existingValue.valid === !!value.valid) {
+                    let parentPath = getParentPathStr(value.path);
+                    // We check for the parent trigger late because getting the parent path is a little bit expensive
+                    if (!config.initialTriggers.parentPaths.has(parentPath)) {
+                        if (PathRouter.isLocalPath(value.path)) {
+                            require("debugbreak")(2);
+                            debugger;
+                        }
+
+                        console.log(`Ignoring duplicate value.`, {
+                            path: value.path,
+                            timeId: value.time.time,
+                            timeIdVersion: value.time.version,
+                            timeIdFull: value.time,
+                            existingValid: existingValue?.valid,
+                            newValid: value.valid,
+                        });
+                        // NOTE: This should be safe. If this causes any problems, it will probably be if we're not the authority on it. In which case, we could move this check so it only happens if we are the authority on it. However, I think even if we aren't the authority, if we already have the exact value and the valid state hasn't changed, we really shouldn't have to ingest it. Watchers should be triggering when they start watching, so they shouldn't need us to trigger them again by re-ingesting it. 
+                        return false;
+                    }
+                }
+            }
+
             if (!PathRouter.isSelfAuthority(value.path)) {
                 notOurValues.push(value);
             } else {
                 ourValues.push(value);
             }
-        }
+            return true;
+        });
         authorityStorage.ingestValues(notOurValues, { doNotArchive });
 
 
@@ -93,7 +122,6 @@ class ValidStateComputer {
             }
         }
 
-
         let initialPrevValidStates: Map<PathValue, boolean | undefined> | undefined = this.capturePrevValidStates(ourValues);
         authorityStorage.ingestValues(ourValues, { doNotArchive });
 

package/src/0-path-value-core/pathValueCore.ts

@@ -431,7 +431,7 @@ class AuthorityPathValueStorage {
     public getValue(path: string): unknown {
         return pathValueSerializer.getPathValue(this.getValueAtTime(path));
     }
-    public getValueExactIgnoreInvalid(path: string, time: Time): PathValue | undefined {
+    public getValueExact(path: string, time: Time): PathValue | undefined {
         let value = this.getValueAtTime(path, time, true);
         if (value && compareTime(value.time, time) === 0) {
             return value;

package/src/4-querysub/Querysub.ts

@@ -1026,8 +1026,6 @@ export class Querysub {
         RequireController.addMapGetModules(async (result, args) => {
             let configObj = args[2] as { signedIdentity: SignedIdentity | undefined } | undefined;
             if (!await isAllowedToSeeSource(configObj?.signedIdentity)) {
-                require("debugbreak")(2);
-                debugger;
                 await isAllowedToSeeSource(configObj?.signedIdentity);
                 //console.log(red(`Not allowed to see source`));
                 for (let [key, value] of Object.entries(result.modules)) {

package/src/diagnostics/logs/IndexedLogs/BufferIndexLogsOptimizationConstants.ts

@@ -13,7 +13,8 @@ export const STREAMING_BLOCK_THRESHOLD = 1024 * 1024;
 
 export const PUBLIC_MOVE_THRESHOLD = timeInHour * 3;
 // Max uncompressed data size
-export const MAX_SINGLE_FILE_DATA = 1024 * 1024 * 512;
+// NOTE: This used to be 512MB, but searching was really slow. I think making this smaller will make searching faster?
+export const MAX_SINGLE_FILE_DATA = 1024 * 1024 * 64;
 
 export const MOVING_TIMEOUT = timeInMinute * 5;
 

package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts

@@ -479,8 +479,7 @@ export class IndexedLogs<T> {
         if (onResultsCallback) {
             interval = setInterval(async () => {
                 updateResultsStats();
-                let results = getFinalResults();
-                let shouldContinue = await onResultsCallback(results);
+                let shouldContinue = await onResultsCallback(getFinalResults());
                 if (!shouldContinue) {
                     if (!results.cancel) {
                         console.log(blue(`Cancelled search on ${this.config.name}`));

package/src/diagnostics/logs/IndexedLogs/moveIndexLogsToPublic.ts

@@ -6,7 +6,7 @@ import { BufferIndex } from "./BufferIndex";
 import { LogStreamer } from "./LogStreamer";
 import { TimeFileTree } from "./TimeFileTree";
 import { blue, green, magenta } from "socket-function/src/formatting/logColors";
-import { formatNumber, formatTime } from "socket-function/src/formatting/format";
+import { formatDateTime, formatNumber, formatTime } from "socket-function/src/formatting/format";
 
 
 
@@ -150,15 +150,15 @@ export async function moveLogsToPublic(config: {
     let groups: typeof initialGroups = [];
     for (let group of initialGroups) {
         let currentSize = 0;
-        
+
         for (let path of group) {
             let size = sizeMap.get(path) || 0;
-            
+
             if (groups.length === 0 || currentSize + size > sizeThreshold) {
                 groups.push([]);
                 currentSize = 0;
             }
-            
+
             groups[groups.length - 1].push(path);
             currentSize += size;
         }
@@ -256,7 +256,7 @@ export async function moveLogsToPublic(config: {
             await localLogs.del(path.fullPath + indexExtension);
         }));
 
-        console.log(green(`(${i + 1}/${groups.length}) Wrote ${encoded.length} log files to public (${formatNumber(encoded.reduce((acc, x) => acc + x.uncompressedSize, 0))}B compressed to ${formatNumber(encoded.reduce((acc, x) => acc + x.compressedSize, 0))}B + ${formatNumber(encoded.reduce((acc, x) => acc + x.index.length, 0))}B index) in ${formatTime(Date.now() - time)}`) + ` (${blue(config.loggerName)})`);
+        console.log(green(`(${i + 1}/${groups.length}) Wrote ${encoded.length} log files to public from ${group.length} input files, ${formatDateTime(startTime)} to ${formatDateTime(endTime)} (${formatNumber(encoded.reduce((acc, x) => acc + x.uncompressedSize, 0))}B compressed to ${formatNumber(encoded.reduce((acc, x) => acc + x.compressedSize, 0))}B + ${formatNumber(encoded.reduce((acc, x) => acc + x.index.length, 0))}B index) in ${formatTime(Date.now() - time)}`) + ` (${blue(config.loggerName)})`);
     }
 
     // Clean up orphaned index files (index files without corresponding data files)

package/src/diagnostics/logs/ObjectDisplay.tsx

@@ -99,7 +99,7 @@ export class ObjectDisplay extends qreact.Component<{
                 <span className={css.maxHeight("70vh").overflowAuto} onClick={log}>
                     <div class={css.relative}>
                         {renderValue(value, [])}
-                        <MeasureHeightCSS />
+                        {/* <MeasureHeightCSS /> */}
                     </div>
                 </span>
             </ShowMore>;