querysub 0.649.0 → 0.651.0

package/overview.md

@@ -0,0 +1,46 @@
+# Querysub Overview
+
+The system is fundamentally a data synchronization system.
+
+## Path values and time
+
+Data exists in path values, which have a path (a structured list of strings) and a value. These are time based: they have the time that they occur at, which is a globally unique time. This time includes a somewhat unique identifier for the server that created the time (best effort — it cannot be relied on, but is usually unique), the time itself, and also a version value that our system uses to create something that happens at the same time, but just an epsilon amount after.
+
+## Locks and validity
+
+Path values also include what are referred to as locks. Locks are essentially a list of reads that were done at the time the path value was created.
+
+Values are evaluated in an eventually consistent manner. A value can be rejected if its locks are rejected. This can mean that the value it read has itself become rejected, or it can mean that the value it read was not the latest value at the time — which we know if we find another value between the creation time of the value that was read and the time that it was read at. That's why it's called a lock: it's a kind of locking of that value between its creation time and the time we read it. Of course, it's a reversed lock — instead of having the writer fail because of the lock, we have the reader fail.
+
+## Synchronization and watchers
+
+Synchronization is very important. We synchronize values between different watchers in a client, and between machines. This is done at a low level in PathWatcher and RemoteWatcher. We have the ability to watch a path, or watch the direct children of a path. There is no recursive watching — if values need to be recursively watched, the watcher has to recursively access more and more each time it receives values, watching more each time.
+
+## Schema
+
+We have a schema system which can simplify accesses and make it so that if values aren't provided, we know whether to default to a primitive value or use a proxy so the code can drill down further.
+
+## Proxy watcher
+
+Most accesses are done using the proxy watcher. This is a system that sits on top of our other systems, letting you write regular JavaScript code to access fields and write to fields as if it were plain JavaScript, with all of the reads and writes being converted to path value reads and writes.
+
+## Client writes, function calls, and predictions
+
+On the client, anything that needs to impact a remote value is mutated by making a special socket function call to the server, asking it to add a function call write — which writes a path value to a specific location. We then have a function runner server that reads from that location, running those functions. The client side also runs these functions, but only as a prediction. That way most actions feel immediate. When the server function runner does eventually run, it updates with the actual results.
+
+## Sharding
+
+The servers are sharded — they are split up over different values. We generally don't do global path hashing. Instead, there are specific paths that a server reads on startup, and it does its hashing by looking at the child key of those paths when they match. These generally match up with lookups, so the hashes are usually the lookup keys. This allows an object's fields to be on the same server — minimizing the number of different servers we access — while still sharding the data.
+
+## Geographic routing
+
+Almost everything has some level of sharding or geographic control. For example, the client side tries to connect to the querysub server — the server that actually adds the function calls and interacts with the client — and it usually connects to the closest one, or the one with the lowest latency. Function runners are also sharded between geographic locations, and the client usually requests one that is close to it.
+
+## Non-core systems
+
+- The binary format of path values, which involves compression, etc.
+- The error notification system.
+- The logging and log search system.
+- The MCP server.
+- QReact, which implements a JSX component renderer using our synchronization system.
+- A machine and service management system.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.649.0",
+  "version": "0.651.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -79,8 +79,8 @@
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
-    "sliftutils": "^1.7.120",
-    "socket-function": "^1.2.33",
+    "sliftutils": "^1.7.124",
+    "socket-function": "^1.2.34",
     "terser": "^5.31.0",
     "typenode": "^6.6.1",
     "typesafecss": "^0.32.0",

package/src/0-path-value-core/PathRouterRouteOverride.ts

@@ -55,6 +55,13 @@ export function getRoutingOverridePart(part: string): {
     };
 }
 
+export function getRoutingOverrideOriginalKey(part: string): string | undefined {
+    if (!part.startsWith(keySpecialIdentifier)) return undefined;
+    let parts = part.split("!");
+    if (parts.length < 4) return undefined;
+    return parts.slice(3).join("!");
+}
+
 export const hasPrefixHash = cacheLimited(1000 * 10,
     (config: { spec: AuthoritySpec, prefixHash: string }) => {
         let { spec, prefixHash } = config;

package/src/deployManager/components/DeployPage.tsx

@@ -265,7 +265,7 @@ export class DeployPage extends qreact.Component {
 
         let liveGitRef = mostCommon(liveFunctions.map(x => x.gitRef)) || "";
         let pendingGitRef = gitInfo?.latestRef || "";
-        let anyUncommitted = !!gitInfo?.uncommitted.length;
+        //let anyUncommitted = !!gitInfo?.uncommitted.length;
 
         return <div className={css.vbox(40).marginTop(20)}>
             <div className={css.hbox(10)}>
@@ -274,12 +274,12 @@ export class DeployPage extends qreact.Component {
                 <button
                     disabled={controller.isAnyLoading()}
                     className={buttonStyle.alignSelf("stretch").center.hsl(240, 70, 50).colorhsl(0, 0, 100)
-                        + (anyUncommitted && css.opacity(0.5))
+                        //+ (anyUncommitted && css.opacity(0.5))
                         + (this.state.isDeploying && css.opacity(0.7))
                     }
-                    title={anyUncommitted && `Must commit uncommitted changes before deploying` || this.state.isDeploying && "Deploying..." || ""}
+                    title={this.state.isDeploying && "Deploying..." || ""}
                     onClick={async () => {
-                        if (anyUncommitted || this.state.isDeploying) return;
+                        if (this.state.isDeploying) return;
 
                         this.state.isDeploying = true;
                         this.state.deployStartTime = Date.now();

package/src/diagnostics/logs/IndexedLogs/MCPIndexedLogsEntry.ts

@@ -20,11 +20,14 @@ import { MCPIndexedLogs, normalizeTime } from "./MCPIndexedLogs";
 import { searchStorageLogsForMCP } from "../../../deployManager/components/storage/storageLogMCPSearch";
 import { getAllNodeIds } from "../../../-f-node-discovery/NodeDiscovery";
 import { NodeCapabilitiesController } from "../../../-g-core-values/NodeCapabilities";
-import { formatTime } from "socket-function/src/formatting/format";
+import { formatDateTime, formatTime } from "socket-function/src/formatting/format";
 import { SocketFunction } from "socket-function/SocketFunction";
+import { sort } from "socket-function/src/misc";
+import { getSuppressionEntries } from "../errorNotifications2/errorNotifications";
 
 const DEFAULT_MCP_HTTP_PORT = 4487;
 const NODE_INFO_TIMEOUT_MS = 5000;
+const DEFAULT_SUPPRESSION_LIMIT = 20;
 
 const PROTOCOL_VERSION = "2025-03-26";
 const SERVER_INFO = { name: "querysub-indexed-logs", version: "0.1.0" };
@@ -97,6 +100,19 @@ Query syntax (case-sensitive substring match against each entry's JSON):
             properties: {},
         },
     },
+    {
+        name: "getSuppressions",
+        description: `List error-suppression entries (patterns that stop matching errors from triggering notifications), sorted most-recently-updated first. There can be hundreds, so results are limited (default ${DEFAULT_SUPPRESSION_LIMIT}); use the optional query filter to narrow.
+
+Returns { total, returned, results } — total is the entry count after filtering, results contains { id, pattern, notes, lastUpdated, created, timeout, expired }. All three times are formatted date strings. expired is present (true) when the entry's timeout has passed, meaning it no longer suppresses anything.`,
+        inputSchema: {
+            type: "object",
+            properties: {
+                limit: { type: "number", default: DEFAULT_SUPPRESSION_LIMIT, description: "Maximum entries to return, most recently updated first." },
+                query: { type: "string", description: "Optional case-insensitive substring filter, matched against id, pattern, and notes." },
+            },
+        },
+    },
 ];
 
 type JsonRpcRequest = {
@@ -184,6 +200,8 @@ async function dispatch(method: string, params: unknown, mcp: MCPIndexedLogs): P
             });
         } else if (toolName === "listNodes") {
             result = await getNodeInfos();
+        } else if (toolName === "getSuppressions") {
+            result = await getSuppressionsForMCP(args as { limit?: number; query?: string });
         } else {
             throw new Error(`Unknown tool ${toolName}`);
         }
@@ -220,6 +238,34 @@ async function getNodeInfos(): Promise<NodeInfo[]> {
     );
 }
 
+async function getSuppressionsForMCP(config: { limit?: number; query?: string }) {
+    let limit = config.limit ?? DEFAULT_SUPPRESSION_LIMIT;
+    let entries = await getSuppressionEntries();
+    let query = config.query?.toLowerCase();
+    if (query) {
+        const q = query;
+        entries = entries.filter(x => (x.id + "\n" + x.pattern + "\n" + (x.notes || "")).toLowerCase().includes(q));
+    }
+    sort(entries, x => -x.lastUpdatedTime);
+    let now = Date.now();
+    let results = entries.slice(0, limit).map(x => {
+        let expired: true | undefined;
+        if (x.timeout < now) {
+            expired = true;
+        }
+        return {
+            id: x.id,
+            pattern: x.pattern,
+            notes: x.notes,
+            lastUpdated: formatDateTime(x.lastUpdatedTime),
+            created: formatDateTime(x.createdTime),
+            timeout: formatDateTime(x.timeout),
+            expired,
+        };
+    });
+    return { total: entries.length, returned: results.length, results };
+}
+
 async function main() {
     let mcp = new MCPIndexedLogs();
 

package/src/diagnostics/logs/TimeRangeSelector.tsx

@@ -27,7 +27,7 @@ export function getTimeRange(): { startTime: number; endTime: number; searchFrom
     };
 }
 
-export class TimeRangeSelector extends qreact.Component {
+export class TimeRangeSelector extends qreact.Component<{ dark?: boolean }> {
     render() {
         const timeRange = getTimeRange();
 
@@ -51,8 +51,12 @@ export class TimeRangeSelector extends qreact.Component {
 
 
         let now = Querysub.nowDelayed(timeInMinute);
+        let containerCss = css.vbox(12).pad2(16).bord2(200, 20, 80).hsl(200, 10, 98);
+        if (this.props.dark) {
+            containerCss = css.vbox(12).pad2(16).bord2(200, 20, 30).hsl(200, 10, 12);
+        }
         return (
-            <div className={css.vbox(12).pad2(16).bord2(200, 20, 80).hsl(200, 10, 98)}>
+            <div className={containerCss}>
                 <div className={css.hbox(12).wrap}>
                     <div className={css.hbox(8).wrap.opacity(0.8).width(200)}>
                         From {formatTime(now - timeRange.startTime)} AGO to {timeRange.endTime > now ? "now" : `${formatTime(now - timeRange.endTime)} AGO`}
@@ -86,7 +90,7 @@ export class TimeRangeSelector extends qreact.Component {
                         }}
                     />
                     <Button
-                        hue={110} onClick={() => {
+                        hue={210} onClick={() => {
                             let now = Date.now();
                             startTimeParam.value = now;
                             endTimeParam.value = undefined;
@@ -95,7 +99,7 @@ export class TimeRangeSelector extends qreact.Component {
                         Set to future data
                     </Button>
                     <Button
-                        hue={110} onClick={() => {
+                        hue={210} onClick={() => {
                             startTimeParam.value = now - timeInHour;
                             endTimeParam.value = now + timeInHour * 2;
                         }}
@@ -103,7 +107,7 @@ export class TimeRangeSelector extends qreact.Component {
                         Set to last hour
                     </Button>
                     <Button
-                        hue={110} onClick={() => {
+                        hue={210} onClick={() => {
                             startTimeParam.value = now - timeInDay;
                             endTimeParam.value = now + timeInHour * 2;
                         }}
@@ -111,7 +115,7 @@ export class TimeRangeSelector extends qreact.Component {
                         Set to last day
                     </Button>
                     <Button
-                        hue={110} onClick={() => {
+                        hue={210} onClick={() => {
                             startTimeParam.value = now - timeInDay * 7;
                             endTimeParam.value = now + timeInHour * 2;
                         }}
@@ -119,12 +123,12 @@ export class TimeRangeSelector extends qreact.Component {
                         Set to last 7 days
                     </Button>
                     {!!(endTimeParam.value || startTimeParam.value) && <Button
-                        hue={110} onClick={resetToLastDay}
+                        hue={210} onClick={resetToLastDay}
                     >
                         Reset
                     </Button>}
                     {(!startTimeParam.value || !endTimeParam.value) && <Button
-                        hue={110}
+                        hue={210}
                         onClick={() => {
                             startTimeParam.value = timeRange.startTime;
                             endTimeParam.value = timeRange.endTime;

package/src/diagnostics/pathAuditer.ts

@@ -394,22 +394,27 @@ async function auditAuthority(nodeId: string, pathsToAudit: { path: string }[],
             continue;
         }
 
+        let responseMissing = !response.time || compareTime(response.time, epochTime) === 0;
+        if (responseMissing) {
+            response = { ...response, valid: true, isTransparent: true };
+        }
         if (ourValue.isTransparent && response.isTransparent) continue;
 
         if (request.time) {
             let ourExact = authorityStorage.getValueExactMaybeRejected(request.path, request.time);
             if (!ourExact || !ourExact.valid) continue;
             if (compareTime(ourExact.time, epochTime) === 0) continue;
+            if (ourExact.isTransparent && responseMissing) continue;
             // The exact value = server does not have
             //      - Send it our value
-            if (!response.time) {
+            if (!response.time || responseMissing) {
                 valuesToSend.push(ourExact);
                 trackSyncAge({
                     path: request.path,
                     ourTimeId: ourExact.time.time,
                     remoteTimeId: undefined,
                     ourValid: ourExact.valid,
-                    remoteValid: response.valid,
+                    remoteValid: undefined,
                     remoteNodeId: nodeId,
                     reason: "Remote is missing our value, sending it to them",
                 });
@@ -471,14 +476,14 @@ async function auditAuthority(nodeId: string, pathsToAudit: { path: string }[],
         }
         // Our latest valid = server does not have & it's latest valid is older than ours
         //      - Send it our value
-        else if (response.valid === undefined && (!response.time || compareTime(ourValue.time, response.time) > 0)) {
+        else if (responseMissing && compareTime(ourValue.time, epochTime) > 0) {
             valuesToSend.push(ourValue);
             trackSyncAge({
                 path: response.path,
                 ourTimeId: ourValue.time.time,
-                remoteTimeId: response.time?.time,
+                remoteTimeId: undefined,
                 ourValid: ourValue.valid ?? false,
-                remoteValid: response.valid,
+                remoteValid: undefined,
                 remoteNodeId: nodeId,
                 reason: "Remote is missing our value, sending it to them",
             });
@@ -535,12 +540,13 @@ class PathAuditerService {
             } else {
                 value = authorityStorage.getValueAtOrBeforeTime(request.path);
             }
+            value = value || createMissingEpochValue(request.path);
             results.push({
                 path: request.path,
-                time: value?.time,
-                valid: value === undefined ? undefined : value.valid,
-                isTransparent: !!value?.isTransparent,
-                event: !!value?.event,
+                time: value.time,
+                valid: value.valid,
+                isTransparent: !!value.isTransparent,
+                event: !!value.event,
             });
         }
         return results;