querysub 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dependency-cruiser.js +304 -0
- package/.eslintrc.js +51 -0
- package/.github/copilot-instructions.md +1 -0
- package/.vscode/settings.json +25 -0
- package/bin/deploy.js +4 -0
- package/bin/function.js +4 -0
- package/bin/server.js +4 -0
- package/costsBenefits.txt +112 -0
- package/deploy.ts +3 -0
- package/inject.ts +1 -0
- package/package.json +60 -0
- package/prompts.txt +54 -0
- package/spec.txt +820 -0
- package/src/-a-archives/archiveCache.ts +913 -0
- package/src/-a-archives/archives.ts +148 -0
- package/src/-a-archives/archivesBackBlaze.ts +792 -0
- package/src/-a-archives/archivesDisk.ts +418 -0
- package/src/-a-archives/copyLocalToBackblaze.ts +24 -0
- package/src/-a-auth/certs.ts +517 -0
- package/src/-a-auth/der.ts +122 -0
- package/src/-a-auth/ed25519.ts +1015 -0
- package/src/-a-auth/node-forge-ed25519.d.ts +17 -0
- package/src/-b-authorities/dnsAuthority.ts +203 -0
- package/src/-b-authorities/emailAuthority.ts +57 -0
- package/src/-c-identity/IdentityController.ts +200 -0
- package/src/-d-trust/NetworkTrust2.ts +150 -0
- package/src/-e-certs/EdgeCertController.ts +288 -0
- package/src/-e-certs/certAuthority.ts +192 -0
- package/src/-f-node-discovery/NodeDiscovery.ts +543 -0
- package/src/-g-core-values/NodeCapabilities.ts +134 -0
- package/src/-g-core-values/oneTimeForward.ts +91 -0
- package/src/-h-path-value-serialize/PathValueSerializer.ts +769 -0
- package/src/-h-path-value-serialize/stringSerializer.ts +176 -0
- package/src/0-path-value-core/LoggingClient.tsx +24 -0
- package/src/0-path-value-core/NodePathAuthorities.ts +978 -0
- package/src/0-path-value-core/PathController.ts +1 -0
- package/src/0-path-value-core/PathValueCommitter.ts +565 -0
- package/src/0-path-value-core/PathValueController.ts +231 -0
- package/src/0-path-value-core/archiveLocks/ArchiveLocks.ts +154 -0
- package/src/0-path-value-core/archiveLocks/ArchiveLocks2.ts +820 -0
- package/src/0-path-value-core/archiveLocks/archiveSnapshots.ts +180 -0
- package/src/0-path-value-core/debugLogs.ts +90 -0
- package/src/0-path-value-core/pathValueArchives.ts +483 -0
- package/src/0-path-value-core/pathValueCore.ts +2217 -0
- package/src/1-path-client/RemoteWatcher.ts +558 -0
- package/src/1-path-client/pathValueClientWatcher.ts +702 -0
- package/src/2-proxy/PathValueProxyWatcher.ts +1857 -0
- package/src/2-proxy/archiveMoveHarness.ts +376 -0
- package/src/2-proxy/garbageCollection.ts +753 -0
- package/src/2-proxy/pathDatabaseProxyBase.ts +37 -0
- package/src/2-proxy/pathValueProxy.ts +139 -0
- package/src/2-proxy/schema2.ts +518 -0
- package/src/3-path-functions/PathFunctionHelpers.ts +129 -0
- package/src/3-path-functions/PathFunctionRunner.ts +619 -0
- package/src/3-path-functions/PathFunctionRunnerMain.ts +67 -0
- package/src/3-path-functions/deployBlock.ts +10 -0
- package/src/3-path-functions/deployCheck.ts +7 -0
- package/src/3-path-functions/deployMain.ts +160 -0
- package/src/3-path-functions/pathFunctionLoader.ts +282 -0
- package/src/3-path-functions/syncSchema.ts +475 -0
- package/src/3-path-functions/tests/functionsTest.ts +135 -0
- package/src/3-path-functions/tests/rejectTest.ts +77 -0
- package/src/4-dom/css.tsx +29 -0
- package/src/4-dom/cssTypes.d.ts +212 -0
- package/src/4-dom/qreact.tsx +2322 -0
- package/src/4-dom/qreactTest.tsx +417 -0
- package/src/4-querysub/Querysub.ts +877 -0
- package/src/4-querysub/QuerysubController.ts +620 -0
- package/src/4-querysub/copyEvent.ts +0 -0
- package/src/4-querysub/permissions.ts +289 -0
- package/src/4-querysub/permissionsShared.ts +1 -0
- package/src/4-querysub/querysubPrediction.ts +525 -0
- package/src/5-diagnostics/FullscreenModal.tsx +67 -0
- package/src/5-diagnostics/GenericFormat.tsx +165 -0
- package/src/5-diagnostics/Modal.tsx +79 -0
- package/src/5-diagnostics/Table.tsx +183 -0
- package/src/5-diagnostics/TimeGrouper.tsx +114 -0
- package/src/5-diagnostics/diskValueAudit.ts +216 -0
- package/src/5-diagnostics/memoryValueAudit.ts +442 -0
- package/src/5-diagnostics/nodeMetadata.ts +135 -0
- package/src/5-diagnostics/qreactDebug.tsx +309 -0
- package/src/5-diagnostics/shared.ts +26 -0
- package/src/5-diagnostics/synchronousLagTracking.ts +47 -0
- package/src/TestController.ts +35 -0
- package/src/allowclient.flag +0 -0
- package/src/bits.ts +86 -0
- package/src/buffers.ts +69 -0
- package/src/config.ts +53 -0
- package/src/config2.ts +48 -0
- package/src/diagnostics/ActionsHistory.ts +56 -0
- package/src/diagnostics/NodeViewer.tsx +503 -0
- package/src/diagnostics/SizeLimiter.ts +62 -0
- package/src/diagnostics/TimeDebug.tsx +18 -0
- package/src/diagnostics/benchmark.ts +139 -0
- package/src/diagnostics/errorLogs/ErrorLogController.ts +515 -0
- package/src/diagnostics/errorLogs/ErrorLogCore.ts +274 -0
- package/src/diagnostics/errorLogs/LogClassifiers.tsx +302 -0
- package/src/diagnostics/errorLogs/LogFilterUI.tsx +84 -0
- package/src/diagnostics/errorLogs/LogNotify.tsx +101 -0
- package/src/diagnostics/errorLogs/LogTimeSelector.tsx +724 -0
- package/src/diagnostics/errorLogs/LogViewer.tsx +757 -0
- package/src/diagnostics/errorLogs/hookErrors.ts +60 -0
- package/src/diagnostics/errorLogs/logFiltering.tsx +149 -0
- package/src/diagnostics/heapTag.ts +13 -0
- package/src/diagnostics/listenOnDebugger.ts +77 -0
- package/src/diagnostics/logs/DiskLoggerPage.tsx +572 -0
- package/src/diagnostics/logs/ObjectDisplay.tsx +165 -0
- package/src/diagnostics/logs/ansiFormat.ts +108 -0
- package/src/diagnostics/logs/diskLogGlobalContext.ts +38 -0
- package/src/diagnostics/logs/diskLogger.ts +305 -0
- package/src/diagnostics/logs/diskShimConsoleLogs.ts +32 -0
- package/src/diagnostics/logs/injectFileLocationToConsole.ts +50 -0
- package/src/diagnostics/logs/logGitHashes.ts +30 -0
- package/src/diagnostics/managementPages.tsx +289 -0
- package/src/diagnostics/periodic.ts +89 -0
- package/src/diagnostics/runSaturationTest.ts +416 -0
- package/src/diagnostics/satSchema.ts +64 -0
- package/src/diagnostics/trackResources.ts +82 -0
- package/src/diagnostics/watchdog.ts +55 -0
- package/src/errors.ts +132 -0
- package/src/forceProduction.ts +3 -0
- package/src/fs.ts +72 -0
- package/src/heapDumps.ts +666 -0
- package/src/https.ts +2 -0
- package/src/inject.ts +1 -0
- package/src/library-components/ATag.tsx +84 -0
- package/src/library-components/Button.tsx +344 -0
- package/src/library-components/ButtonSelector.tsx +64 -0
- package/src/library-components/DropdownCustom.tsx +151 -0
- package/src/library-components/DropdownSelector.tsx +32 -0
- package/src/library-components/Input.tsx +334 -0
- package/src/library-components/InputLabel.tsx +198 -0
- package/src/library-components/InputPicker.tsx +125 -0
- package/src/library-components/LazyComponent.tsx +62 -0
- package/src/library-components/MeasureHeightCSS.tsx +48 -0
- package/src/library-components/MeasuredDiv.tsx +47 -0
- package/src/library-components/ShowMore.tsx +51 -0
- package/src/library-components/SyncedController.ts +171 -0
- package/src/library-components/TimeRangeSelector.tsx +407 -0
- package/src/library-components/URLParam.ts +263 -0
- package/src/library-components/colors.tsx +14 -0
- package/src/library-components/drag.ts +114 -0
- package/src/library-components/icons.tsx +692 -0
- package/src/library-components/niceStringify.ts +50 -0
- package/src/library-components/renderToString.ts +52 -0
- package/src/misc/PromiseRace.ts +101 -0
- package/src/misc/color.ts +30 -0
- package/src/misc/getParentProcessId.cs +53 -0
- package/src/misc/getParentProcessId.ts +53 -0
- package/src/misc/hash.ts +83 -0
- package/src/misc/ipPong.js +13 -0
- package/src/misc/networking.ts +2 -0
- package/src/misc/random.ts +45 -0
- package/src/misc.ts +19 -0
- package/src/noserverhotreload.flag +0 -0
- package/src/path.ts +226 -0
- package/src/persistentLocalStore.ts +37 -0
- package/src/promise.ts +15 -0
- package/src/server.ts +73 -0
- package/src/src.d.ts +1 -0
- package/src/test/heapProcess.ts +36 -0
- package/src/test/mongoSatTest.tsx +55 -0
- package/src/test/satTest.ts +193 -0
- package/src/test/test.tsx +552 -0
- package/src/zip.ts +92 -0
- package/src/zipThreaded.ts +106 -0
- package/src/zipThreadedWorker.js +19 -0
- package/tsconfig.json +27 -0
- package/yarnSpec.txt +56 -0
|
@@ -0,0 +1,525 @@
|
|
|
1
|
+
import { cacheJSONArgsEqual, cacheWeak, lazy } from "socket-function/src/caching";
|
|
2
|
+
import { pathValueCommitter } from "../0-path-value-core/PathValueController";
|
|
3
|
+
import { CLIENTSIDE_PREDICT_LEEWAY, MAX_ACCEPTED_CHANGE_AGE, MAX_CHANGE_AGE, PathValue, ReadLock, Time, authorityStorage, compareTime, debugTime, getCreatorId, getNextTime, lockToCallback, registerIsOurPrediction, timeMinusEpsilon } from "../0-path-value-core/pathValueCore";
|
|
4
|
+
import { remoteWatcher } from "../1-path-client/RemoteWatcher";
|
|
5
|
+
import { proxyWatcher, atomicObjectRead } from "../2-proxy/PathValueProxyWatcher";
|
|
6
|
+
import { getPathFromProxy, getProxyPath } from "../2-proxy/pathValueProxy";
|
|
7
|
+
import { CallSpec, FunctionResult, FunctionSpec, debugCallSpec, functionSchema, overrideCurrentCall } from "../3-path-functions/PathFunctionRunner";
|
|
8
|
+
import { getModuleFromConfig, setGitURLMapping } from "../3-path-functions/pathFunctionLoader";
|
|
9
|
+
import { logErrors } from "../errors";
|
|
10
|
+
import { getParentPathStr, getPathFromStr, getPathStr1, getPathStr2 } from "../path";
|
|
11
|
+
import { Querysub, QuerysubController, baseAddCall, callWaitOn, querysubNodeId } from "./QuerysubController";
|
|
12
|
+
import { Benchmark } from "../diagnostics/benchmark";
|
|
13
|
+
import { parseArgs } from "../3-path-functions/PathFunctionHelpers";
|
|
14
|
+
import { runInSerial } from "socket-function/src/batching";
|
|
15
|
+
import { green, magenta, red, yellow } from "socket-function/src/formatting/logColors";
|
|
16
|
+
import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
|
|
17
|
+
|
|
18
|
+
import { setFlag } from "socket-function/require/compileFlags";
|
|
19
|
+
import cbor from "cbor-x";
|
|
20
|
+
import { FunctionMetadata } from "../3-path-functions/syncSchema";
|
|
21
|
+
import { isNode, nextId, sort } from "socket-function/src/misc";
|
|
22
|
+
setFlag(require, "cbor-x", "allowclient", true);
|
|
23
|
+
const cborEncoder = lazy(() => new cbor.Encoder({ structuredClone: true }));
|
|
24
|
+
|
|
25
|
+
// TODO: I think our use of filePath, moduleId, etc are wrong here? We give pathFunctionLoader the filePath,
|
|
26
|
+
// but we really should just give it the moduleId, or... even just avoid calling it altogether, as it doesn't
|
|
27
|
+
// do too much for us if we already have the fully resolved path...
|
|
28
|
+
// - Although using it DOES allow permissions checks to work nicely, so, eh... maybe it is fine to use pathFunctionLoader?
|
|
29
|
+
const addModuleToLoader = cacheJSONArgsEqual(async (spec: FunctionSpec): Promise<void> => {
|
|
30
|
+
let nodeId = await querysubNodeId();
|
|
31
|
+
if (!nodeId) throw new Error("No querysub node found");
|
|
32
|
+
let path = await QuerysubController.nodes[nodeId].getModulePath({ functionSpec: spec });
|
|
33
|
+
setGitURLMapping({ spec, resolvedPath: path });
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
export function getCallResultPath(call: CallSpec) {
|
|
38
|
+
return getProxyPath(() => functionSchema()[call.DomainName].PathFunctionRunner[call.ModuleId].Results[call.CallId]);
|
|
39
|
+
}
|
|
40
|
+
export function getCallResult(call: CallSpec) {
|
|
41
|
+
return functionSchema()[call.DomainName].PathFunctionRunner[call.ModuleId].Results[call.CallId];
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** Force predictions to run in the trigger order, so they can resolve an be added before
|
|
45
|
+
* the next predictions. Also, to preserve call order.
|
|
46
|
+
*/
|
|
47
|
+
const predictRunCommitLoop = runInSerial((run: () => Promise<PredictResult>) => run());
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
interface DelayedCall {
|
|
51
|
+
call: CallSpec;
|
|
52
|
+
time: number;
|
|
53
|
+
commit: () => void;
|
|
54
|
+
cancel: () => void;
|
|
55
|
+
|
|
56
|
+
result?: PredictResult;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
let callPending: DelayedCall[] = [];
|
|
60
|
+
function flushUpToIncluding(time: number) {
|
|
61
|
+
while (callPending.length > 0) {
|
|
62
|
+
if (callPending[0].time > time) break;
|
|
63
|
+
let call = callPending.shift()!;
|
|
64
|
+
call.commit();
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export function flushDelayedFunctions() {
|
|
69
|
+
while (callPending.length > 0) {
|
|
70
|
+
flushUpToIncluding(callPending[callPending.length - 1].time);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export const addCall = runInSerial(async function addCall(call: CallSpec, metadata: FunctionMetadata) {
|
|
75
|
+
const nodeId = await querysubNodeId();
|
|
76
|
+
if (!nodeId) throw new Error("No querysub node found");
|
|
77
|
+
|
|
78
|
+
// TODO: Use some heuristics to determine if we should predict the call or not (ex, based on predicted time
|
|
79
|
+
// to run, current load, latency to server, time for server to run it, time to sync the necessary data, etc)
|
|
80
|
+
// TODO: Allow some flags to tell us NOT to predict a call
|
|
81
|
+
let predict = Querysub.PREDICT_CALLS;
|
|
82
|
+
if (metadata.nopredict) {
|
|
83
|
+
predict = false;
|
|
84
|
+
}
|
|
85
|
+
let cancel = () => { };
|
|
86
|
+
|
|
87
|
+
if (predict) {
|
|
88
|
+
let delayed = metadata.delayCommit && !isNode();
|
|
89
|
+
let predictObj = predictCall({ call, localOnly: delayed });
|
|
90
|
+
cancel = predictObj.cancel;
|
|
91
|
+
callWaitOn(call.CallId, predictObj.predictPromise);
|
|
92
|
+
if (metadata.delayCommit && !isNode()) {
|
|
93
|
+
let hasRun = false;
|
|
94
|
+
let delayedCall: DelayedCall = {
|
|
95
|
+
call,
|
|
96
|
+
time: Date.now(),
|
|
97
|
+
commit() {
|
|
98
|
+
if (hasRun) return;
|
|
99
|
+
hasRun = true;
|
|
100
|
+
logErrors(baseAddCall(call, nodeId, cancel, `call (delayed from ${debugTime(call.runAtTime)})`));
|
|
101
|
+
},
|
|
102
|
+
cancel() { predictObj.cancel(); },
|
|
103
|
+
};
|
|
104
|
+
callPending.push(delayedCall);
|
|
105
|
+
setTimeout(() => flushUpToIncluding(delayedCall.time), Querysub.DELAY_COMMIT_DELAY);
|
|
106
|
+
logErrors(predictObj.predictPromise.then(async (prediction): Promise<void> => {
|
|
107
|
+
prediction = { ...prediction };
|
|
108
|
+
const resultPrefix = getProxyPath(() => functionSchema()[call.DomainName].PathFunctionRunner[call.ModuleId].Results);
|
|
109
|
+
prediction.writes = prediction.writes.filter(write => !write.path.startsWith(resultPrefix));
|
|
110
|
+
delayedCall.result = prediction;
|
|
111
|
+
|
|
112
|
+
let index = callPending.indexOf(delayedCall);
|
|
113
|
+
if (index === -1) return;
|
|
114
|
+
let ourWritePaths = new Set(prediction.writes.map(x => x.path));
|
|
115
|
+
function candidateCouldBeChangedByWritesSimilarToOurs(candidate: PredictResult) {
|
|
116
|
+
return (
|
|
117
|
+
candidate.writes.some(write => ourWritePaths.has(write.path))
|
|
118
|
+
|| candidate.writes.some(write => candidate.readPaths.has(write.path))
|
|
119
|
+
|| candidate.writes.some(write =>
|
|
120
|
+
candidate.readParentPaths.has(getParentPathStr(write.path))
|
|
121
|
+
|| candidate.readParentPaths.has(write.path)
|
|
122
|
+
)
|
|
123
|
+
);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
// NOTE: Technically... we can remove a write if:
|
|
127
|
+
// - latest.writes.paths is a superset of candidate.writes.paths
|
|
128
|
+
// - (candidate <= intermediate < latest).all(x =>
|
|
129
|
+
// test replace writes of x with previous values of x, as time of x
|
|
130
|
+
// rerun writes up to and including latest
|
|
131
|
+
// verify final resolved writes are equivalent as the final writes as if we had the candidate write
|
|
132
|
+
// )
|
|
133
|
+
// HOWEVER, the last condition is overly arduous to implement, wildly inefficient to calculate, so
|
|
134
|
+
// we can simplify it to a more conservative (removes less candidates, but still is never wrong):
|
|
135
|
+
// - (candidate < intermediate < latest).all(x => none of x.write.paths are in latest.writes.paths and none of x.read.paths are in latest.writes.paths)
|
|
136
|
+
// - candidate.writes.paths is a subset of latest.writes.paths
|
|
137
|
+
// - test replace writes of x with previous values of x, as time of x
|
|
138
|
+
// rerun latest
|
|
139
|
+
// verify writes2.writes === writes.writes (paths and values)
|
|
140
|
+
// NOTE: See "delayCommit" in syncSchema for the risks of delaying commits.
|
|
141
|
+
|
|
142
|
+
// Find the first write that is a candidate (due to our simplified condition this will only be 1).
|
|
143
|
+
let curIndex = index - 1;
|
|
144
|
+
while (curIndex >= 0) {
|
|
145
|
+
let candidatePred = callPending[curIndex].result;
|
|
146
|
+
if (!candidatePred) {
|
|
147
|
+
// NOTE: This shouldn't happen, but if it does... we could just stop searching here, and not
|
|
148
|
+
// remove any predictions.
|
|
149
|
+
throw new Error("Predictions finished out of order.");
|
|
150
|
+
}
|
|
151
|
+
if (candidateCouldBeChangedByWritesSimilarToOurs(candidatePred)) {
|
|
152
|
+
break;
|
|
153
|
+
}
|
|
154
|
+
curIndex--;
|
|
155
|
+
}
|
|
156
|
+
// Nothing to remove, we don't overlap any other writes
|
|
157
|
+
if (curIndex < 0) {
|
|
158
|
+
return;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
let candidate = callPending[curIndex];
|
|
162
|
+
let candidatePred = candidate.result;
|
|
163
|
+
if (!candidatePred) {
|
|
164
|
+
throw new Error("Predictions finished out of order.");
|
|
165
|
+
}
|
|
166
|
+
// candidate writes have to be a strict subset of our writes, otherwise there is no way we will clobber the candidate.
|
|
167
|
+
if (candidatePred.writes.some(write => !ourWritePaths.has(write.path))) return;
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
let debugName = `[redundant check]|${call.DomainName}.${call.FunctionId}`;
|
|
171
|
+
// NOTE: Run right before the call, so we skip it's writes, but use everything else. We could also use
|
|
172
|
+
// candidate.time + epsilon, because none of the values between call and candidate matter (they don't write
|
|
173
|
+
// to any paths either read from).
|
|
174
|
+
let beforeCall = { ...call };
|
|
175
|
+
beforeCall.runAtTime = timeMinusEpsilon(getPredictTime(call.runAtTime));
|
|
176
|
+
let withoutCandidateResult = await getCallWrites({
|
|
177
|
+
call: beforeCall,
|
|
178
|
+
debugName,
|
|
179
|
+
overrides: candidatePred.replacedWriteValues
|
|
180
|
+
});
|
|
181
|
+
|
|
182
|
+
function areWritesEqual(writes: PathValue[], writes2: PathValue[]) {
|
|
183
|
+
if (writes.length !== writes2.length) return false;
|
|
184
|
+
// Write order should probably be the same anyways, so... it is far to compare in order
|
|
185
|
+
for (let i = 0; i < writes.length; i++) {
|
|
186
|
+
let write = writes[i];
|
|
187
|
+
let write2 = writes2[i];
|
|
188
|
+
if (write.path !== write2.path) return false;
|
|
189
|
+
if (pathValueSerializer.compareValuePaths(write, write2) !== 0) {
|
|
190
|
+
return false;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
return true;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
if (!areWritesEqual(withoutCandidateResult.writes, prediction.writes)) {
|
|
197
|
+
if (Querysub.DEBUG_CALLS) {
|
|
198
|
+
// Failed to collapse delayed functions, because one of the previous predictions was used in the new predicted function. This is valid, but slower than necessary. If pure atomic behavior is not required (which it presumably isn't required, as delayCommit is already being used), try to rewrite the function to be in the form of a set, instead of an add. For example, "function add(count) { x += count; }" => "function set(value) { x = value; }" (calling with set(x + count)).
|
|
199
|
+
console.log(`[Querysub] ${yellow("CANNOT DELAY COLLAPSE")} ${debugCallSpec(candidate.call)} `);
|
|
200
|
+
}
|
|
201
|
+
return;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
let index2 = callPending.indexOf(candidate);
|
|
205
|
+
if (index2 < 0) return;
|
|
206
|
+
// At this point... we KNOW:
|
|
207
|
+
// a) There are no writes between us and `candidate` (writes are never inserted in the middle, only removed,
|
|
208
|
+
// so this condition will hold despite our delay)
|
|
209
|
+
// b) `candidate` is not required for `call` to run
|
|
210
|
+
// c) `call` replaces all values inside of `candidate`
|
|
211
|
+
// Which means, `candidate is entirely redunant, and can be removed!
|
|
212
|
+
|
|
213
|
+
if (Querysub.DEBUG_CALLS) {
|
|
214
|
+
console.log(`[Querysub] ${magenta("removed redundant call")} ${green(debugCallSpec(candidate.call))} @${debugTime(candidate.call.runAtTime)}`);
|
|
215
|
+
}
|
|
216
|
+
callPending.splice(index2, 1);
|
|
217
|
+
// Cancel as well, in case it isn't redundant, so we can see the issue immediately.
|
|
218
|
+
candidate.cancel();
|
|
219
|
+
|
|
220
|
+
// Move our time back, otherwise repeated delays can result in not committing writes for
|
|
221
|
+
// a long time (indefinitely really), which can result in a lot of lost work, and
|
|
222
|
+
// has little speed benefit (comitting a write every 5 seconds is fine).
|
|
223
|
+
delayedCall.time = candidate.time;
|
|
224
|
+
}));
|
|
225
|
+
|
|
226
|
+
if (Querysub.DEBUG_CALLS) {
|
|
227
|
+
console.log(`[Querysub] ${magenta("delaying")} ${green(debugCallSpec(call))} @${debugTime(call.runAtTime)}`);
|
|
228
|
+
}
|
|
229
|
+
return;
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
await flushDelayedFunctions();
|
|
234
|
+
// NOTE: NO MORE calls can be queued here, because addCall is run in serial, so... we don't need
|
|
235
|
+
// to worry about new calls with times > call.runAtTime (and if it happens we'll get a fairly
|
|
236
|
+
// clear error clientside, which will show up in a notification, so it is a very safe assumption).
|
|
237
|
+
await baseAddCall(call, nodeId, cancel, "call");
|
|
238
|
+
});
|
|
239
|
+
|
|
240
|
+
function getPredictTime(time: Time) {
|
|
241
|
+
return { time: time.time, version: -2, creatorId: time.creatorId };
|
|
242
|
+
}
|
|
243
|
+
const predictionLockVersion = -2344282313;
|
|
244
|
+
export function isOurPrediction(value: PathValue) {
|
|
245
|
+
return value.locks.length === 1 && value.locks[0].startTime.version === predictionLockVersion;
|
|
246
|
+
}
|
|
247
|
+
registerIsOurPrediction(isOurPrediction);
|
|
248
|
+
|
|
249
|
+
interface PredictResult {
|
|
250
|
+
readPaths: Set<string>;
|
|
251
|
+
readParentPaths: Set<string>;
|
|
252
|
+
writes: PathValue[];
|
|
253
|
+
/** Writes that have been replaced AT the write time of the prediction */
|
|
254
|
+
replacedWriteValues: PathValue[];
|
|
255
|
+
}
|
|
256
|
+
function predictCall(config: {
|
|
257
|
+
call: CallSpec;
|
|
258
|
+
localOnly?: boolean;
|
|
259
|
+
overrides?: PathValue[];
|
|
260
|
+
}): {
|
|
261
|
+
cancel: () => void;
|
|
262
|
+
predictPromise: Promise<PredictResult>;
|
|
263
|
+
} {
|
|
264
|
+
let call = config.call;
|
|
265
|
+
let pathResultWrite = getCallResultPath(call);
|
|
266
|
+
Benchmark.onStartPredictCall(pathResultWrite);
|
|
267
|
+
let functionResult: FunctionResult = {
|
|
268
|
+
// Special prediction values (we don't even really need to put anything here, but it
|
|
269
|
+
// might as well have the right format, in case a bug causes it to be used somewhere)
|
|
270
|
+
runCount: -1,
|
|
271
|
+
timeTaken: -1,
|
|
272
|
+
evalTime: 0,
|
|
273
|
+
};
|
|
274
|
+
|
|
275
|
+
// NOTE: We don't cascade predictions, instead only depending on the call succeeding. This is because if we depend on writes,
|
|
276
|
+
// we will be invalidated before we get the actual call result, which results in going back in time, a delay, then
|
|
277
|
+
// having the actual call results, and so jumping forward again. It is better to just wait, so our latest value
|
|
278
|
+
// becomes the latest call value, when we receive it.
|
|
279
|
+
let predictResultWrite: PathValue = {
|
|
280
|
+
path: pathResultWrite,
|
|
281
|
+
value: functionResult,
|
|
282
|
+
locks: [
|
|
283
|
+
{
|
|
284
|
+
// We use a special version, so we don't have to worry about this actually existing. We don't really want
|
|
285
|
+
// a lock on startTime, we just want the range lock.
|
|
286
|
+
startTime: { time: call.runAtTime.time, version: predictionLockVersion, creatorId: call.runAtTime.creatorId },
|
|
287
|
+
// NOTE: The version is special, and used to detect prediction rejections in some logging.
|
|
288
|
+
// NOTE: We need to set our end time enough in the future so errors cause us to be rejected
|
|
289
|
+
endTime: { time: call.runAtTime.time + MAX_CHANGE_AGE, version: Number.MAX_SAFE_INTEGER, creatorId: call.runAtTime.creatorId },
|
|
290
|
+
path: pathResultWrite,
|
|
291
|
+
// Technically we are depending on a future time, so... we are depending on ourself? And our value isn't undefined.
|
|
292
|
+
readIsTranparent: false,
|
|
293
|
+
keepRejectedUntil: call.runAtTime.time + CLIENTSIDE_PREDICT_LEEWAY
|
|
294
|
+
}
|
|
295
|
+
],
|
|
296
|
+
lockCount: 1,
|
|
297
|
+
// We right our result BEFORE the actual result, so it will be clobbered (and before our read lock!)
|
|
298
|
+
// when the result arrives (although... it will ALSO be rejected, so... this might be less important...)
|
|
299
|
+
// (Also, more importantly, this ha to be UNIQUE, otherwise time locks break!)
|
|
300
|
+
time: getPredictTime(call.runAtTime),
|
|
301
|
+
valid: true,
|
|
302
|
+
event: true,
|
|
303
|
+
};
|
|
304
|
+
let predictLocks: ReadLock[] = [
|
|
305
|
+
{
|
|
306
|
+
startTime: predictResultWrite.time,
|
|
307
|
+
endTime: predictResultWrite.time,
|
|
308
|
+
path: pathResultWrite,
|
|
309
|
+
readIsTranparent: false,
|
|
310
|
+
}
|
|
311
|
+
];
|
|
312
|
+
let predictions: PredictResult | undefined;
|
|
313
|
+
let predictPromise = predictRunCommitLoop(async () => {
|
|
314
|
+
if (Querysub.DEBUG_PREDICTIONS) {
|
|
315
|
+
console.log(magenta(`Start predict call`), `${call.DomainName}.${call.FunctionId}`);
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
if (!config.localOnly) {
|
|
319
|
+
// Watch the result, so we know when our prediction is rejected (which will be as soon as the result
|
|
320
|
+
// has a real value).
|
|
321
|
+
remoteWatcher.watchLatest({ paths: [pathResultWrite], parentPaths: [] });
|
|
322
|
+
}
|
|
323
|
+
let debugName = `[predict]|${call.DomainName}.${call.FunctionId}`;
|
|
324
|
+
|
|
325
|
+
let dryRunResult: {
|
|
326
|
+
writes: PathValue[];
|
|
327
|
+
readPaths: Set<string>;
|
|
328
|
+
readParentPaths: Set<string>;
|
|
329
|
+
};
|
|
330
|
+
try {
|
|
331
|
+
dryRunResult = await getCallWrites({ call, debugName, overrides: config.overrides });
|
|
332
|
+
} catch (e) {
|
|
333
|
+
|
|
334
|
+
if (!pathValueSerializer.getPathValue(authorityStorage.getValueAtTime(pathResultWrite, undefined))) {
|
|
335
|
+
console.log(`Skipping prediction for ${debugName} due to error running predictive call. Likely just an out of order error.`, e);
|
|
336
|
+
} else {
|
|
337
|
+
// NOTE: This case happens a lot, because of how we handle locks. We don't receive locked values, and so
|
|
338
|
+
// we assume all values have no locks, and only keep the latest. This is usually fine, but... if we lose
|
|
339
|
+
// the race to predict the function against the server updating it, it is likely our prediction will now
|
|
340
|
+
// be running before the latest write. In which case (as we don't really store write history), we will read undefined.
|
|
341
|
+
// This isn't accurate, but... our write WILL almost certainly be wrong (as the value changed), so we don't log here.
|
|
342
|
+
}
|
|
343
|
+
return {
|
|
344
|
+
writes: [],
|
|
345
|
+
readPaths: new Set(),
|
|
346
|
+
readParentPaths: new Set(),
|
|
347
|
+
replacedWriteValues: [],
|
|
348
|
+
};
|
|
349
|
+
}
|
|
350
|
+
predictions = {
|
|
351
|
+
writes: dryRunResult.writes,
|
|
352
|
+
readPaths: dryRunResult.readPaths,
|
|
353
|
+
readParentPaths: dryRunResult.readParentPaths,
|
|
354
|
+
replacedWriteValues: dryRunResult.writes.map(write => {
|
|
355
|
+
let path = write.path;
|
|
356
|
+
let newWrite: PathValue = authorityStorage.getValueAtTime(write.path, write.time) || {
|
|
357
|
+
path, valid: true, time: write.time, locks: [], lockCount: 0, value: undefined, canGCValue: true, isTransparent: true,
|
|
358
|
+
};
|
|
359
|
+
newWrite = { ...newWrite };
|
|
360
|
+
// Use a time very slightly after, so it clobbers the write, and any writes before it, but none after.
|
|
361
|
+
newWrite.time = write.time;
|
|
362
|
+
write.time = {
|
|
363
|
+
time: write.time.time,
|
|
364
|
+
version: write.time.version,
|
|
365
|
+
creatorId: write.time.creatorId + 1,
|
|
366
|
+
};
|
|
367
|
+
return newWrite;
|
|
368
|
+
}),
|
|
369
|
+
};
|
|
370
|
+
|
|
371
|
+
for (let value of predictions.writes) {
|
|
372
|
+
value.source = debugName;
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
// If we already received the actual call, we can't add the prediction, as... the invalidation
|
|
376
|
+
// code won't properly immediately reject our prediction, as we are not the authority on the
|
|
377
|
+
// path, so it treats it as source of truth.
|
|
378
|
+
if (authorityStorage.getValueAtTime(pathResultWrite, undefined)?.value) {
|
|
379
|
+
if (Querysub.DEBUG_PREDICTIONS) {
|
|
380
|
+
console.log(magenta(`Abort predict call, already received call result`), `${call.DomainName}.${call.FunctionId}`);
|
|
381
|
+
}
|
|
382
|
+
return predictions;
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
// We commit slightly different writes, for lock invalidation. Our caller doesn't need / doesn't want this
|
|
386
|
+
// extra write, so we just don't return it.
|
|
387
|
+
// NOTE: I guess we technically don't need to actually write to the result, but... might as well?
|
|
388
|
+
predictions.writes.push(predictResultWrite);
|
|
389
|
+
// Update all locks to use the same locks as predictResultWrite
|
|
390
|
+
// Ensure we use the predict time, otherwise these times will be reused when the call actually happens,
|
|
391
|
+
// potentially with different values, which breaks all of our ReadLocks!
|
|
392
|
+
for (let predict of predictions.writes) {
|
|
393
|
+
predict.time = predictResultWrite.time;
|
|
394
|
+
predict.locks = predictLocks;
|
|
395
|
+
predict.lockCount = predictLocks.length;
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
pathValueCommitter.ingestValues(predictions.writes, undefined, undefined);
|
|
399
|
+
|
|
400
|
+
// NOTE: We have to forcefully setup the lock watches, as we are no the authority on these paths
|
|
401
|
+
// so it is odd for us to do this! It works out though, because the path is unique enough
|
|
402
|
+
// that we don't have to worry about any other valid states coming in for our lock.
|
|
403
|
+
// - Also, once we set up this lock, we should get full reject cascade support.
|
|
404
|
+
lockToCallback.watchLock(predictResultWrite.locks[0], [predictResultWrite]);
|
|
405
|
+
lockToCallback.watchLock(predictLocks[0], predictions.writes);
|
|
406
|
+
|
|
407
|
+
if (Querysub.DEBUG_PREDICTIONS) {
|
|
408
|
+
console.log(magenta(`Finished predict call`), `${call.DomainName}.${call.FunctionId}`);
|
|
409
|
+
}
|
|
410
|
+
return predictions;
|
|
411
|
+
});
|
|
412
|
+
logErrors(predictPromise);
|
|
413
|
+
|
|
414
|
+
let didCancel = false;
|
|
415
|
+
// ALWAYS reject the prediction eventually, in case the function runner server is down.
|
|
416
|
+
// - ALSO, once it goes back up, the write will be too far in the future, and our lock won't be rejected.
|
|
417
|
+
// And don't really want endTime to be infinitely in the future, as then cleanup code has a harder time
|
|
418
|
+
// removing the lock, as then in theory it can always be rejected, no matter how long we wait.
|
|
419
|
+
setTimeout(rejectPrediction, Querysub.PREDICTION_MAX_LIFESPAN);
|
|
420
|
+
function rejectPrediction() {
|
|
421
|
+
if (didCancel) return;
|
|
422
|
+
didCancel = true;
|
|
423
|
+
// Reject our predictions, as the call likely never got committed, so it will never be written
|
|
424
|
+
pathValueCommitter.ingestValidStates([{
|
|
425
|
+
path: pathResultWrite,
|
|
426
|
+
isValid: false,
|
|
427
|
+
time: predictResultWrite.locks[0].startTime,
|
|
428
|
+
isTransparent: false,
|
|
429
|
+
}]);
|
|
430
|
+
}
|
|
431
|
+
setTimeout(auditPrediction, Querysub.PREDICTION_MAX_LIFESPAN);
|
|
432
|
+
function auditPrediction() {
|
|
433
|
+
if (didCancel) return;
|
|
434
|
+
if (Querysub.AUDIT_PREDICTIONS && predictions) {
|
|
435
|
+
const afterTime = { time: call.runAtTime.time, version: Number.MAX_SAFE_INTEGER, creatorId: 0 };
|
|
436
|
+
// Clone predictions, to strip symbols
|
|
437
|
+
let predictionsCopy = cborEncoder().decode(cborEncoder().encode(predictions)) as typeof predictions;
|
|
438
|
+
for (let predict of predictionsCopy.writes ?? []) {
|
|
439
|
+
if (predict.path === pathResultWrite) continue;
|
|
440
|
+
let finalValueObj = authorityStorage.getValueAtTime(predict.path, afterTime);
|
|
441
|
+
if (!authorityStorage.isSynced(predict.path)) continue;
|
|
442
|
+
|
|
443
|
+
let finalValue = pathValueSerializer.getPathValue(finalValueObj);
|
|
444
|
+
let predictValue = pathValueSerializer.getPathValue(predict);
|
|
445
|
+
|
|
446
|
+
let finalBuffer = cborEncoder().encode(finalValue);
|
|
447
|
+
let predictBuffer = cborEncoder().encode(predictValue);
|
|
448
|
+
// If they are different, warn
|
|
449
|
+
if (!finalBuffer.equals(predictBuffer)) {
|
|
450
|
+
// NOTE: This MIGHT be due to delayCommit on another function. We can't know when to force flush
|
|
451
|
+
// delayCommit values, and so we only force flush if other (non-delayCommit) functions are committed
|
|
452
|
+
// in the same schema, which will be wrong if data is accessed cross schema.
|
|
453
|
+
// - If this happens, and is a problem, we COULD handle it, but... it is very difficult, as we would need
|
|
454
|
+
// to evaluate the other function, detect it conflicts with a delayCommit functions, then run the delayCommit
|
|
455
|
+
// function, then re-evaluated the conflicted prediction (which is a lot of communication and reruns
|
|
456
|
+
// between a lot of systems... especially because in a second this will resolve by the server
|
|
457
|
+
// clobbering our prediction).
|
|
458
|
+
// - AND, we can't commit the pending calls until we sort out the order, so this necessarily requires
|
|
459
|
+
// slowing down commits if we are delaying other calls.
|
|
460
|
+
console.warn(`${red("Prediction was wrong")}: ${getPathFromStr(predict.path).join(".")} predict != finalValue`, predictValue, finalValue);
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
return {
|
|
467
|
+
cancel: rejectPrediction,
|
|
468
|
+
predictPromise,
|
|
469
|
+
};
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
export async function getCallWrites(config: {
|
|
473
|
+
debugName: string;
|
|
474
|
+
call: CallSpec;
|
|
475
|
+
overrides?: PathValue[];
|
|
476
|
+
}) {
|
|
477
|
+
let { call, debugName } = config;
|
|
478
|
+
let { functionSpec } = await proxyWatcher.commitFunction({
|
|
479
|
+
watchFunction: function getModuleConfig() {
|
|
480
|
+
//let moduleConfig = atomicObjectRead(functionSchema()[call.DomainName].PathFunctionRunner[call.ModuleId].Module);
|
|
481
|
+
// if (!(call.DomainName in functionSchema())) {
|
|
482
|
+
// throw new Error(`Domain not found ${call.DomainName}`);
|
|
483
|
+
// }
|
|
484
|
+
let domainObject = functionSchema()[call.DomainName];
|
|
485
|
+
if (!(call.ModuleId in domainObject.PathFunctionRunner)) {
|
|
486
|
+
throw new Error(`Module not found ${call.DomainName}.${call.ModuleId}`);
|
|
487
|
+
}
|
|
488
|
+
let moduleObject = domainObject.PathFunctionRunner[call.ModuleId];
|
|
489
|
+
if (!(call.FunctionId in moduleObject.Sources)) {
|
|
490
|
+
throw new Error(`Function not found ${call.DomainName}.${call.ModuleId}.${call.FunctionId}`);
|
|
491
|
+
}
|
|
492
|
+
let functionSpec = atomicObjectRead(moduleObject.Sources[call.FunctionId]);
|
|
493
|
+
return { functionSpec };
|
|
494
|
+
}
|
|
495
|
+
});
|
|
496
|
+
if (!functionSpec) {
|
|
497
|
+
throw new Error(`Function not found ${call.DomainName}.${call.ModuleId}.${call.FunctionId}`);
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
await addModuleToLoader(functionSpec);
|
|
501
|
+
|
|
502
|
+
let module = await getModuleFromConfig(functionSpec);
|
|
503
|
+
let exportPath = getPathFromStr(functionSpec.exportPathStr);
|
|
504
|
+
let exportObj = module.exports;
|
|
505
|
+
for (let path of exportPath) {
|
|
506
|
+
exportObj = exportObj[path];
|
|
507
|
+
}
|
|
508
|
+
let baseFunction = exportObj as Function;
|
|
509
|
+
|
|
510
|
+
return await proxyWatcher.dryRunFull({
|
|
511
|
+
debugName,
|
|
512
|
+
runAtTime: call.runAtTime,
|
|
513
|
+
overrideAllowLockDomainsPrefixes: [getPathStr1(call.DomainName)],
|
|
514
|
+
// We are going to clobber locks anyways, so there is no reason to create them
|
|
515
|
+
unsafeNoLocks: true,
|
|
516
|
+
overrides: config.overrides,
|
|
517
|
+
nestedCalls: "inline",
|
|
518
|
+
watchFunction() {
|
|
519
|
+
overrideCurrentCall(call, () => {
|
|
520
|
+
let args = parseArgs(call);
|
|
521
|
+
baseFunction(...args);
|
|
522
|
+
});
|
|
523
|
+
},
|
|
524
|
+
});
|
|
525
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import preact from "preact";
|
|
2
|
+
import { qreact } from "../4-dom/qreact";
|
|
3
|
+
import { Button } from "../library-components/Button";
|
|
4
|
+
import { Modal } from "./Modal";
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
export class FullscreenModal extends preact.Component<{
|
|
8
|
+
parentState?: { open: boolean };
|
|
9
|
+
onCancel?: () => void;
|
|
10
|
+
style?: preact.JSX.CSSProperties;
|
|
11
|
+
outerStyle?: preact.JSX.CSSProperties;
|
|
12
|
+
}> {
|
|
13
|
+
render() {
|
|
14
|
+
let { parentState } = this.props;
|
|
15
|
+
return (
|
|
16
|
+
<Modal>
|
|
17
|
+
<div style={{ display: "none" }}>
|
|
18
|
+
<Button hotkeys={["Global+Escape"]} onClick={() => {
|
|
19
|
+
if (parentState) parentState.open = false;
|
|
20
|
+
this.props.onCancel?.();
|
|
21
|
+
}}>Close</Button>
|
|
22
|
+
</div>
|
|
23
|
+
<div
|
|
24
|
+
style={{
|
|
25
|
+
position: "fixed",
|
|
26
|
+
top: 0,
|
|
27
|
+
left: 0,
|
|
28
|
+
width: "100vw",
|
|
29
|
+
height: "100vh",
|
|
30
|
+
background: "hsla(0, 0%, 30%, 0.5)",
|
|
31
|
+
padding: 100,
|
|
32
|
+
display: "flex",
|
|
33
|
+
alignItems: "center",
|
|
34
|
+
justifyContent: "center",
|
|
35
|
+
overflow: "auto",
|
|
36
|
+
cursor: "pointer",
|
|
37
|
+
...this.props.outerStyle,
|
|
38
|
+
}}
|
|
39
|
+
onClick={e => {
|
|
40
|
+
if (qreact.isTarget(e)) {
|
|
41
|
+
if (parentState) parentState.open = false;
|
|
42
|
+
this.props.onCancel?.();
|
|
43
|
+
}
|
|
44
|
+
}}
|
|
45
|
+
>
|
|
46
|
+
<div
|
|
47
|
+
style={{
|
|
48
|
+
background: "hsl(0, 0%, 100%)",
|
|
49
|
+
padding: 20,
|
|
50
|
+
color: "hsl(0, 0%, 7%)",
|
|
51
|
+
cursor: "default",
|
|
52
|
+
width: "100%",
|
|
53
|
+
display: "flex",
|
|
54
|
+
flexDirection: "column",
|
|
55
|
+
gap: 10,
|
|
56
|
+
maxHeight: "calc(100% - 200px)",
|
|
57
|
+
overflow: "auto",
|
|
58
|
+
...this.props.style
|
|
59
|
+
}}
|
|
60
|
+
>
|
|
61
|
+
{this.props.children}
|
|
62
|
+
</div>
|
|
63
|
+
</div>
|
|
64
|
+
</Modal>
|
|
65
|
+
);
|
|
66
|
+
}
|
|
67
|
+
}
|