querysub 0.319.0 → 0.321.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.319.0",
+  "version": "0.321.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/4-deploy/edgeNodes.ts

@@ -7,7 +7,7 @@ import { getArchivesBackblazePublic } from "../-a-archives/archivesBackBlaze";
 import { nestArchives } from "../-a-archives/archives";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { delay, runInSerial, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
-import { compare, compareArray, isNodeTrue, sort, timeInMinute } from "socket-function/src/misc";
+import { compare, compareArray, isNodeTrue, sort, timeInMinute, timeInSecond } from "socket-function/src/misc";
 import { cacheLimited, lazy } from "socket-function/src/caching";
 import { canHaveChildren } from "socket-function/src/types";
 import { hostArchives } from "../-b-authorities/cdnAuthority";
@@ -25,6 +25,7 @@ import { onEdgeNodesChanged } from "./edgeBootstrap";
 import { startEdgeNotifier } from "./edgeClientWatcher";
 import { getGitRefLive, getGitURLLive } from "./git";
 import { DeployProgress } from "./deployFunctions";
+import { PromiseObj } from "../promise";
 
 const UPDATE_POLL_INTERVAL = timeInMinute * 15;
 const DEAD_NODE_COUNT_THRESHOLD = 15;
@@ -117,19 +118,27 @@ export async function registerEdgeNode(config: {
 
     if (!isLocal() && !noSyncing()) {
         let loadedHash = "";
+        let firstLoad = new PromiseObj<void>();
         Querysub.createWatcher(() => {
             let hash = deploySchema()[getDomain()].deploy.live.hash;
             if (hash === loadedHash || !Querysub.isAllSynced()) return;
 
             loadedHash = hash;
             void Promise.resolve().then(async () => {
-                await loadEntryPointsByHash({
-                    host,
-                    entryPaths: config.entryPaths,
-                    hash,
-                });
+                try {
+                    await loadEntryPointsByHash({
+                        host,
+                        entryPaths: config.entryPaths,
+                        hash,
+                    });
+                } finally {
+                    firstLoad.resolve();
+                }
             });
         });
+        console.log(`Waiting for entry point to finish loading for live hash before registering edge node`);
+        await firstLoad.promise;
+        console.log(`Finished waiting for entry point to finish loading for live hash before registering edge node`);
     } else {
         let gitHash = await getGitRefLive();
 
@@ -217,7 +226,7 @@ const startUpdateLoop = lazy(async () => {
     startEdgeNotifier();
     SocketFunction.expose(EdgeNodeController);
     await getEdgeNodeConfigURL();
-    await runInfinitePollCallAtStart(UPDATE_POLL_INTERVAL, updateLoop);
+    await runInfinitePollCallAtStart(UPDATE_POLL_INTERVAL * (1 + Math.random() * 0.1), updateLoop);
 });
 
 async function updateLoop() {
@@ -292,6 +301,26 @@ async function updateEdgeNodesFile() {
 
     console.log(magenta(`Updating ${edgeNodeIndexFile} with ${JSON.stringify(newEdgeNodeIndex)}`));
     await edgeNodeStorage.set(edgeNodeIndexFile, Buffer.from(JSON.stringify(newEdgeNodeIndex)));
+    await delay(Math.random() * 1000);
+    async function verify() {
+        let buffer = await edgeNodeStorage.get(edgeNodeIndexFile);
+        let index = JSON.parse(buffer!.toString()) as EdgeNodesIndex;
+        let ownNodeId = getOwnNodeId();
+        return index.edgeNodes.some(x => x.nodeId === ownNodeId);
+    }
+    if (registeredEdgeNode) {
+        try {
+            if (!await verify()) {
+                console.error(`Found own our node wasn't in the edge node index file, trying to update again in a bit`);
+                await delay(Math.random() * 1000 + timeInSecond * 10);
+                await updateEdgeNodesFile();
+            }
+        } catch (e) {
+            console.error(`Failed to verify update to edge node index file due to an error, trying to update again in a bit`, { error: e });
+            await delay(Math.random() * 1000 + timeInSecond * 10);
+            await updateEdgeNodesFile();
+        }
+    }
 }
 
 export async function preloadUI(hash: string, progress?: DeployProgress) {

package/src/4-querysub/Querysub.ts

@@ -16,7 +16,7 @@ import { SocketFunction } from "socket-function/SocketFunction";
 import { isHotReloading, watchFilesAndTriggerHotReloading } from "socket-function/hot/HotReloadController";
 import { RequireController, setRequireBootRequire } from "socket-function/require/RequireController";
 import { cache, cacheLimited, lazy } from "socket-function/src/caching";
-import { getOwnMachineId, getThreadKeyCert, verifyMachineIdForPublicKey } from "../-a-auth/certs";
+import { getOwnMachineId, getOwnThreadId, getThreadKeyCert, verifyMachineIdForPublicKey } from "../-a-auth/certs";
 import { getHostedIP, getSNICerts, publishMachineARecords } from "../-e-certs/EdgeCertController";
 import { LOCAL_DOMAIN, nodePathAuthority } from "../0-path-value-core/NodePathAuthorities";
 import { debugCoreMode, registerGetCompressNetwork, encodeParentFilter, registerGetCompressDisk, authorityStorage } from "../0-path-value-core/pathValueCore";
@@ -519,7 +519,7 @@ export class Querysub {
 
     public static getOwnNodeId = getOwnNodeId;
     public static getSelfNodeId = getOwnNodeId;
-    public static getOwnThreadId = getOwnNodeId;
+    public static getOwnThreadId = getOwnThreadId;
 
     /** Set ClientWatcher.DEBUG_SOURCES to true for to be populated */
     public static getTriggerReason() {

package/src/deployManager/components/ServicesListPage.tsx

@@ -29,6 +29,12 @@ export class ServicesListPage extends qreact.Component {
 
         let getMachineInfo = cache((machineId: string) => controller.getMachineInfo(machineId));
 
+        let keyCounts = new Map<string, number>();
+        for (let [serviceId, config] of services) {
+            let key = config?.parameters.key || "";
+            keyCounts.set(key, (keyCounts.get(key) || 0) + 1);
+        }
+
         return <div className={css.vbox(16)}>
             <div className={css.hbox(12).wrap}>
                 <h2 className={css.flexGrow(1)}>Services</h2>
@@ -87,6 +93,7 @@ export class ServicesListPage extends qreact.Component {
                         return acc + (machineInfo?.services[serviceId]?.totalTimesLaunched || 0);
                     }, 0);
                     let unknown = config.machineIds.length - runningMachines.length - failingMachines.length - missingMachines.length;
+                    let duplicateKey = (keyCounts.get(config.parameters.key || "") || 0) > 1;
                     return <div className={css.hbox(10)}>
                         <Anchor noStyles key={serviceId}
                             values={[currentViewParam.getOverride("service-detail"), selectedServiceIdParam.getOverride(serviceId)]}
@@ -102,7 +109,14 @@ export class ServicesListPage extends qreact.Component {
                             <div className={css.hbox(12)}>
                                 <div className={css.vbox(4).flexGrow(1)}>
                                     <div className={css.fontSize(14).boldStyle}>{config.info.title}</div>
-                                    <div>{config.parameters.key}</div>
+                                    <div className={css
+                                        + (duplicateKey
+                                            && css.hsl(0, 50, 50).colorhsl(0, 0, 95).pad2(4, 2))
+                                    }
+                                        title={duplicateKey ? "Duplicate key" : undefined}
+                                    >
+                                        {config.parameters.key}
+                                    </div>
                                     <div>
                                         {config.machineIds.length} configured {failingMachines.length > 0 && `(${failingMachines.length} failing)`} {missingMachines.length > 0 && `(${missingMachines.length} machine hasn't run service yet)`} {unknown > 0 && `(${unknown} unknown)`} • {totalLaunches} launches •
                                         Deploy: {config.parameters.deploy ? "enabled" : "disabled"}

package/src/deployManager/machineSchema.ts

@@ -189,7 +189,14 @@ export class MachineServiceControllerBase {
     public async setServiceConfigs(configs: ServiceConfig[]) {
         let newMachines = new Set<string>();
         let oldMachines = new Set<string>();
+        let usedKeys = new Set<string>();
         for (let config of configs) {
+            if (usedKeys.has(config.parameters.key)) {
+                console.warn(`Duplicate key: ${JSON.stringify(config.parameters.key)}. Not allowed, as this will break things! We are unduplicating the key.`);
+                config.parameters.key += `-${Math.random().toString().slice(2)}`;
+            }
+            usedKeys.add(config.parameters.key);
+
             let serviceId = config.serviceId;
             config.info.lastUpdatedTime = Date.now();
             let serviceConfig = await serviceConfigs.get(serviceId);

package/src/diagnostics/logs/FastArchiveViewer.tsx

@@ -611,12 +611,12 @@ export class FastArchiveViewer<T> extends qreact.Component<{
                         {!this.state.finished && <LoaderAurora />}
                         {this.limitedScanCount > 0 && (
                             <div className={infoDisplay(60).boldStyle}>
-                                {formatNumber(this.limitedScanCount)} scanned logs were rate limited. We do not track how many were ignored, there might be infinite missing logs.
+                                {formatNumber(this.limitedScanCount)} scanned logs were rate limited at a file level. We do not track how many were ignored, there might be infinite missing logs.
                             </div>
                         )}
                         {this.limitedMatchCount > 0 && (
                             <div className={infoDisplay(0).boldStyle}>
-                                {formatNumber(this.limitedMatchCount)} matched logs were rate limited. We do not track how many were ignored, there might be infinite missing logs.
+                                {formatNumber(this.limitedMatchCount)} matched logs were rate limited at a file level. We do not track how many were ignored, there might be infinite missing logs.
                             </div>
                         )}
                         {this.state.pendingSyncInitializations > 0 && (

package/src/diagnostics/logs/errorNotifications/ErrorSuppressionUI.tsx

@@ -95,24 +95,7 @@ export class ErrorSuppressionUI extends qreact.Component<{
                 {this.state.matchedInput.trim() && <span className={css.opacity(0.6).fontSize(12).flexShrink0}>
                     matches {formatNumber(previewMatchCount)} (in top 1000)
                 </span>}
-                <Button onClick={() => {
-                    let value = this.state.matchedInput;
-                    this.state.matchedInput = "";
-                    void Querysub.onCommitFinished(async () => {
-                        await controller.setSuppressionEntry.promise({
-                            key: nextId(),
-                            match: value,
-                            comment: "",
-                            lastUpdateTime: Date.now(),
-                            expiresAt: NOT_AN_ERROR_EXPIRE_TIME,
-                        });
-                        Querysub.commit(() => {
-                            this.props.rerunFilters();
-                        });
-                    });
-                }}>
-                    Not an error
-                </Button>
+
                 <Button
                     onClick={() => {
                         let value = this.state.matchedInput;
@@ -134,6 +117,24 @@ export class ErrorSuppressionUI extends qreact.Component<{
                 >
                     Fixed
                 </Button>
+                <Button onClick={() => {
+                    let value = this.state.matchedInput;
+                    this.state.matchedInput = "";
+                    void Querysub.onCommitFinished(async () => {
+                        await controller.setSuppressionEntry.promise({
+                            key: nextId(),
+                            match: value,
+                            comment: "",
+                            lastUpdateTime: Date.now(),
+                            expiresAt: NOT_AN_ERROR_EXPIRE_TIME,
+                        });
+                        Querysub.commit(() => {
+                            this.props.rerunFilters();
+                        });
+                    });
+                }}>
+                    Not an error
+                </Button>
             </div>
 
             <div className={css.vbox(8).fillWidth.overflowAuto.maxHeight("30vh")}>

package/src/diagnostics/logs/lifeCycleAnalysis/spec.md

@@ -4,12 +4,16 @@ Very small amount of data
     https://127-0-0-1.querysubtest.com:7007/?hot&enableLogs&page=login&filter=%22431%22&showingmanagement&endTime=1755140880000&startTime=1754950020000&managementpage=LogViewer2
 
 
+4) WTF... our http server is trying to run the queue. But we told it not to... Ugh...
 
 5) Update all services, and move them to that machine
 5) Verify the hezner server can run the site well
 6) Take down our digital ocean server
 7) Destroy our digital ocean server
 
+9) Add a filter to JUST see rate limited logs (by clicking on the button. which just searches for the special text)
+    - say "click here to view rate limited logs"
+
 10) "Started Listening" isn't being logged?
     - https://127-0-0-1.querysubtest.com:7007/?enableLogs&page=login&showingmanagement&endTime=1757835685102.667&managementpage=LogViewer2&machineview=service-detail&startTime=1757745685102.667&serviceId=service-1756340309836&filter=__machineId%20%3D%20a794fbcf7b104c68%20%26%20Edge
     - Or... maybe logs are lost SOMETIMES, and ALWAYS when we kill the server? Although... that would mean we have multiple issues. Ugh...