querysub 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (169) hide show
  1. package/.dependency-cruiser.js +304 -0
  2. package/.eslintrc.js +51 -0
  3. package/.github/copilot-instructions.md +1 -0
  4. package/.vscode/settings.json +25 -0
  5. package/bin/deploy.js +4 -0
  6. package/bin/function.js +4 -0
  7. package/bin/server.js +4 -0
  8. package/costsBenefits.txt +112 -0
  9. package/deploy.ts +3 -0
  10. package/inject.ts +1 -0
  11. package/package.json +60 -0
  12. package/prompts.txt +54 -0
  13. package/spec.txt +820 -0
  14. package/src/-a-archives/archiveCache.ts +913 -0
  15. package/src/-a-archives/archives.ts +148 -0
  16. package/src/-a-archives/archivesBackBlaze.ts +792 -0
  17. package/src/-a-archives/archivesDisk.ts +418 -0
  18. package/src/-a-archives/copyLocalToBackblaze.ts +24 -0
  19. package/src/-a-auth/certs.ts +517 -0
  20. package/src/-a-auth/der.ts +122 -0
  21. package/src/-a-auth/ed25519.ts +1015 -0
  22. package/src/-a-auth/node-forge-ed25519.d.ts +17 -0
  23. package/src/-b-authorities/dnsAuthority.ts +203 -0
  24. package/src/-b-authorities/emailAuthority.ts +57 -0
  25. package/src/-c-identity/IdentityController.ts +200 -0
  26. package/src/-d-trust/NetworkTrust2.ts +150 -0
  27. package/src/-e-certs/EdgeCertController.ts +288 -0
  28. package/src/-e-certs/certAuthority.ts +192 -0
  29. package/src/-f-node-discovery/NodeDiscovery.ts +543 -0
  30. package/src/-g-core-values/NodeCapabilities.ts +134 -0
  31. package/src/-g-core-values/oneTimeForward.ts +91 -0
  32. package/src/-h-path-value-serialize/PathValueSerializer.ts +769 -0
  33. package/src/-h-path-value-serialize/stringSerializer.ts +176 -0
  34. package/src/0-path-value-core/LoggingClient.tsx +24 -0
  35. package/src/0-path-value-core/NodePathAuthorities.ts +978 -0
  36. package/src/0-path-value-core/PathController.ts +1 -0
  37. package/src/0-path-value-core/PathValueCommitter.ts +565 -0
  38. package/src/0-path-value-core/PathValueController.ts +231 -0
  39. package/src/0-path-value-core/archiveLocks/ArchiveLocks.ts +154 -0
  40. package/src/0-path-value-core/archiveLocks/ArchiveLocks2.ts +820 -0
  41. package/src/0-path-value-core/archiveLocks/archiveSnapshots.ts +180 -0
  42. package/src/0-path-value-core/debugLogs.ts +90 -0
  43. package/src/0-path-value-core/pathValueArchives.ts +483 -0
  44. package/src/0-path-value-core/pathValueCore.ts +2217 -0
  45. package/src/1-path-client/RemoteWatcher.ts +558 -0
  46. package/src/1-path-client/pathValueClientWatcher.ts +702 -0
  47. package/src/2-proxy/PathValueProxyWatcher.ts +1857 -0
  48. package/src/2-proxy/archiveMoveHarness.ts +376 -0
  49. package/src/2-proxy/garbageCollection.ts +753 -0
  50. package/src/2-proxy/pathDatabaseProxyBase.ts +37 -0
  51. package/src/2-proxy/pathValueProxy.ts +139 -0
  52. package/src/2-proxy/schema2.ts +518 -0
  53. package/src/3-path-functions/PathFunctionHelpers.ts +129 -0
  54. package/src/3-path-functions/PathFunctionRunner.ts +619 -0
  55. package/src/3-path-functions/PathFunctionRunnerMain.ts +67 -0
  56. package/src/3-path-functions/deployBlock.ts +10 -0
  57. package/src/3-path-functions/deployCheck.ts +7 -0
  58. package/src/3-path-functions/deployMain.ts +160 -0
  59. package/src/3-path-functions/pathFunctionLoader.ts +282 -0
  60. package/src/3-path-functions/syncSchema.ts +475 -0
  61. package/src/3-path-functions/tests/functionsTest.ts +135 -0
  62. package/src/3-path-functions/tests/rejectTest.ts +77 -0
  63. package/src/4-dom/css.tsx +29 -0
  64. package/src/4-dom/cssTypes.d.ts +212 -0
  65. package/src/4-dom/qreact.tsx +2322 -0
  66. package/src/4-dom/qreactTest.tsx +417 -0
  67. package/src/4-querysub/Querysub.ts +877 -0
  68. package/src/4-querysub/QuerysubController.ts +620 -0
  69. package/src/4-querysub/copyEvent.ts +0 -0
  70. package/src/4-querysub/permissions.ts +289 -0
  71. package/src/4-querysub/permissionsShared.ts +1 -0
  72. package/src/4-querysub/querysubPrediction.ts +525 -0
  73. package/src/5-diagnostics/FullscreenModal.tsx +67 -0
  74. package/src/5-diagnostics/GenericFormat.tsx +165 -0
  75. package/src/5-diagnostics/Modal.tsx +79 -0
  76. package/src/5-diagnostics/Table.tsx +183 -0
  77. package/src/5-diagnostics/TimeGrouper.tsx +114 -0
  78. package/src/5-diagnostics/diskValueAudit.ts +216 -0
  79. package/src/5-diagnostics/memoryValueAudit.ts +442 -0
  80. package/src/5-diagnostics/nodeMetadata.ts +135 -0
  81. package/src/5-diagnostics/qreactDebug.tsx +309 -0
  82. package/src/5-diagnostics/shared.ts +26 -0
  83. package/src/5-diagnostics/synchronousLagTracking.ts +47 -0
  84. package/src/TestController.ts +35 -0
  85. package/src/allowclient.flag +0 -0
  86. package/src/bits.ts +86 -0
  87. package/src/buffers.ts +69 -0
  88. package/src/config.ts +53 -0
  89. package/src/config2.ts +48 -0
  90. package/src/diagnostics/ActionsHistory.ts +56 -0
  91. package/src/diagnostics/NodeViewer.tsx +503 -0
  92. package/src/diagnostics/SizeLimiter.ts +62 -0
  93. package/src/diagnostics/TimeDebug.tsx +18 -0
  94. package/src/diagnostics/benchmark.ts +139 -0
  95. package/src/diagnostics/errorLogs/ErrorLogController.ts +515 -0
  96. package/src/diagnostics/errorLogs/ErrorLogCore.ts +274 -0
  97. package/src/diagnostics/errorLogs/LogClassifiers.tsx +302 -0
  98. package/src/diagnostics/errorLogs/LogFilterUI.tsx +84 -0
  99. package/src/diagnostics/errorLogs/LogNotify.tsx +101 -0
  100. package/src/diagnostics/errorLogs/LogTimeSelector.tsx +724 -0
  101. package/src/diagnostics/errorLogs/LogViewer.tsx +757 -0
  102. package/src/diagnostics/errorLogs/hookErrors.ts +60 -0
  103. package/src/diagnostics/errorLogs/logFiltering.tsx +149 -0
  104. package/src/diagnostics/heapTag.ts +13 -0
  105. package/src/diagnostics/listenOnDebugger.ts +77 -0
  106. package/src/diagnostics/logs/DiskLoggerPage.tsx +572 -0
  107. package/src/diagnostics/logs/ObjectDisplay.tsx +165 -0
  108. package/src/diagnostics/logs/ansiFormat.ts +108 -0
  109. package/src/diagnostics/logs/diskLogGlobalContext.ts +38 -0
  110. package/src/diagnostics/logs/diskLogger.ts +305 -0
  111. package/src/diagnostics/logs/diskShimConsoleLogs.ts +32 -0
  112. package/src/diagnostics/logs/injectFileLocationToConsole.ts +50 -0
  113. package/src/diagnostics/logs/logGitHashes.ts +30 -0
  114. package/src/diagnostics/managementPages.tsx +289 -0
  115. package/src/diagnostics/periodic.ts +89 -0
  116. package/src/diagnostics/runSaturationTest.ts +416 -0
  117. package/src/diagnostics/satSchema.ts +64 -0
  118. package/src/diagnostics/trackResources.ts +82 -0
  119. package/src/diagnostics/watchdog.ts +55 -0
  120. package/src/errors.ts +132 -0
  121. package/src/forceProduction.ts +3 -0
  122. package/src/fs.ts +72 -0
  123. package/src/heapDumps.ts +666 -0
  124. package/src/https.ts +2 -0
  125. package/src/inject.ts +1 -0
  126. package/src/library-components/ATag.tsx +84 -0
  127. package/src/library-components/Button.tsx +344 -0
  128. package/src/library-components/ButtonSelector.tsx +64 -0
  129. package/src/library-components/DropdownCustom.tsx +151 -0
  130. package/src/library-components/DropdownSelector.tsx +32 -0
  131. package/src/library-components/Input.tsx +334 -0
  132. package/src/library-components/InputLabel.tsx +198 -0
  133. package/src/library-components/InputPicker.tsx +125 -0
  134. package/src/library-components/LazyComponent.tsx +62 -0
  135. package/src/library-components/MeasureHeightCSS.tsx +48 -0
  136. package/src/library-components/MeasuredDiv.tsx +47 -0
  137. package/src/library-components/ShowMore.tsx +51 -0
  138. package/src/library-components/SyncedController.ts +171 -0
  139. package/src/library-components/TimeRangeSelector.tsx +407 -0
  140. package/src/library-components/URLParam.ts +263 -0
  141. package/src/library-components/colors.tsx +14 -0
  142. package/src/library-components/drag.ts +114 -0
  143. package/src/library-components/icons.tsx +692 -0
  144. package/src/library-components/niceStringify.ts +50 -0
  145. package/src/library-components/renderToString.ts +52 -0
  146. package/src/misc/PromiseRace.ts +101 -0
  147. package/src/misc/color.ts +30 -0
  148. package/src/misc/getParentProcessId.cs +53 -0
  149. package/src/misc/getParentProcessId.ts +53 -0
  150. package/src/misc/hash.ts +83 -0
  151. package/src/misc/ipPong.js +13 -0
  152. package/src/misc/networking.ts +2 -0
  153. package/src/misc/random.ts +45 -0
  154. package/src/misc.ts +19 -0
  155. package/src/noserverhotreload.flag +0 -0
  156. package/src/path.ts +226 -0
  157. package/src/persistentLocalStore.ts +37 -0
  158. package/src/promise.ts +15 -0
  159. package/src/server.ts +73 -0
  160. package/src/src.d.ts +1 -0
  161. package/src/test/heapProcess.ts +36 -0
  162. package/src/test/mongoSatTest.tsx +55 -0
  163. package/src/test/satTest.ts +193 -0
  164. package/src/test/test.tsx +552 -0
  165. package/src/zip.ts +92 -0
  166. package/src/zipThreaded.ts +106 -0
  167. package/src/zipThreadedWorker.js +19 -0
  168. package/tsconfig.json +27 -0
  169. package/yarnSpec.txt +56 -0
@@ -0,0 +1,525 @@
1
+ import { cacheJSONArgsEqual, cacheWeak, lazy } from "socket-function/src/caching";
2
+ import { pathValueCommitter } from "../0-path-value-core/PathValueController";
3
+ import { CLIENTSIDE_PREDICT_LEEWAY, MAX_ACCEPTED_CHANGE_AGE, MAX_CHANGE_AGE, PathValue, ReadLock, Time, authorityStorage, compareTime, debugTime, getCreatorId, getNextTime, lockToCallback, registerIsOurPrediction, timeMinusEpsilon } from "../0-path-value-core/pathValueCore";
4
+ import { remoteWatcher } from "../1-path-client/RemoteWatcher";
5
+ import { proxyWatcher, atomicObjectRead } from "../2-proxy/PathValueProxyWatcher";
6
+ import { getPathFromProxy, getProxyPath } from "../2-proxy/pathValueProxy";
7
+ import { CallSpec, FunctionResult, FunctionSpec, debugCallSpec, functionSchema, overrideCurrentCall } from "../3-path-functions/PathFunctionRunner";
8
+ import { getModuleFromConfig, setGitURLMapping } from "../3-path-functions/pathFunctionLoader";
9
+ import { logErrors } from "../errors";
10
+ import { getParentPathStr, getPathFromStr, getPathStr1, getPathStr2 } from "../path";
11
+ import { Querysub, QuerysubController, baseAddCall, callWaitOn, querysubNodeId } from "./QuerysubController";
12
+ import { Benchmark } from "../diagnostics/benchmark";
13
+ import { parseArgs } from "../3-path-functions/PathFunctionHelpers";
14
+ import { runInSerial } from "socket-function/src/batching";
15
+ import { green, magenta, red, yellow } from "socket-function/src/formatting/logColors";
16
+ import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
17
+
18
+ import { setFlag } from "socket-function/require/compileFlags";
19
+ import cbor from "cbor-x";
20
+ import { FunctionMetadata } from "../3-path-functions/syncSchema";
21
+ import { isNode, nextId, sort } from "socket-function/src/misc";
22
+ setFlag(require, "cbor-x", "allowclient", true);
23
+ const cborEncoder = lazy(() => new cbor.Encoder({ structuredClone: true }));
24
+
25
+ // TODO: I think our use of filePath, moduleId, etc are wrong here? We give pathFunctionLoader the filePath,
26
+ // but we really should just give it the moduleId, or... even just avoid calling it altogether, as it doesn't
27
+ // do too much for us if we already have the fully resolved path...
28
+ // - Although using it DOES allow permissions checks to work nicely, so, eh... maybe it is fine to use pathFunctionLoader?
29
+ const addModuleToLoader = cacheJSONArgsEqual(async (spec: FunctionSpec): Promise<void> => {
30
+ let nodeId = await querysubNodeId();
31
+ if (!nodeId) throw new Error("No querysub node found");
32
+ let path = await QuerysubController.nodes[nodeId].getModulePath({ functionSpec: spec });
33
+ setGitURLMapping({ spec, resolvedPath: path });
34
+ });
35
+
36
+
37
+ export function getCallResultPath(call: CallSpec) {
38
+ return getProxyPath(() => functionSchema()[call.DomainName].PathFunctionRunner[call.ModuleId].Results[call.CallId]);
39
+ }
40
+ export function getCallResult(call: CallSpec) {
41
+ return functionSchema()[call.DomainName].PathFunctionRunner[call.ModuleId].Results[call.CallId];
42
+ }
43
+
44
+ /** Force predictions to run in the trigger order, so they can resolve an be added before
45
+ * the next predictions. Also, to preserve call order.
46
+ */
47
+ const predictRunCommitLoop = runInSerial((run: () => Promise<PredictResult>) => run());
48
+
49
+
50
+ interface DelayedCall {
51
+ call: CallSpec;
52
+ time: number;
53
+ commit: () => void;
54
+ cancel: () => void;
55
+
56
+ result?: PredictResult;
57
+ }
58
+
59
+ let callPending: DelayedCall[] = [];
60
+ function flushUpToIncluding(time: number) {
61
+ while (callPending.length > 0) {
62
+ if (callPending[0].time > time) break;
63
+ let call = callPending.shift()!;
64
+ call.commit();
65
+ }
66
+ }
67
+
68
+ export function flushDelayedFunctions() {
69
+ while (callPending.length > 0) {
70
+ flushUpToIncluding(callPending[callPending.length - 1].time);
71
+ }
72
+ }
73
+
74
+ export const addCall = runInSerial(async function addCall(call: CallSpec, metadata: FunctionMetadata) {
75
+ const nodeId = await querysubNodeId();
76
+ if (!nodeId) throw new Error("No querysub node found");
77
+
78
+ // TODO: Use some heuristics to determine if we should predict the call or not (ex, based on predicted time
79
+ // to run, current load, latency to server, time for server to run it, time to sync the necessary data, etc)
80
+ // TODO: Allow some flags to tell us NOT to predict a call
81
+ let predict = Querysub.PREDICT_CALLS;
82
+ if (metadata.nopredict) {
83
+ predict = false;
84
+ }
85
+ let cancel = () => { };
86
+
87
+ if (predict) {
88
+ let delayed = metadata.delayCommit && !isNode();
89
+ let predictObj = predictCall({ call, localOnly: delayed });
90
+ cancel = predictObj.cancel;
91
+ callWaitOn(call.CallId, predictObj.predictPromise);
92
+ if (metadata.delayCommit && !isNode()) {
93
+ let hasRun = false;
94
+ let delayedCall: DelayedCall = {
95
+ call,
96
+ time: Date.now(),
97
+ commit() {
98
+ if (hasRun) return;
99
+ hasRun = true;
100
+ logErrors(baseAddCall(call, nodeId, cancel, `call (delayed from ${debugTime(call.runAtTime)})`));
101
+ },
102
+ cancel() { predictObj.cancel(); },
103
+ };
104
+ callPending.push(delayedCall);
105
+ setTimeout(() => flushUpToIncluding(delayedCall.time), Querysub.DELAY_COMMIT_DELAY);
106
+ logErrors(predictObj.predictPromise.then(async (prediction): Promise<void> => {
107
+ prediction = { ...prediction };
108
+ const resultPrefix = getProxyPath(() => functionSchema()[call.DomainName].PathFunctionRunner[call.ModuleId].Results);
109
+ prediction.writes = prediction.writes.filter(write => !write.path.startsWith(resultPrefix));
110
+ delayedCall.result = prediction;
111
+
112
+ let index = callPending.indexOf(delayedCall);
113
+ if (index === -1) return;
114
+ let ourWritePaths = new Set(prediction.writes.map(x => x.path));
115
+ function candidateCouldBeChangedByWritesSimilarToOurs(candidate: PredictResult) {
116
+ return (
117
+ candidate.writes.some(write => ourWritePaths.has(write.path))
118
+ || candidate.writes.some(write => candidate.readPaths.has(write.path))
119
+ || candidate.writes.some(write =>
120
+ candidate.readParentPaths.has(getParentPathStr(write.path))
121
+ || candidate.readParentPaths.has(write.path)
122
+ )
123
+ );
124
+ }
125
+
126
+ // NOTE: Technically... we can remove a write if:
127
+ // - latest.writes.paths is a superset of candidate.writes.paths
128
+ // - (candidate <= intermediate < latest).all(x =>
129
+ // test replace writes of x with previous values of x, as time of x
130
+ // rerun writes up to and including latest
131
+ // verify final resolved writes are equivalent as the final writes as if we had the candidate write
132
+ // )
133
+ // HOWEVER, the last condition is overly arduous to implement, wildly inefficient to calculate, so
134
+ // we can simplify it to a more conservative (removes less candidates, but still is never wrong):
135
+ // - (candidate < intermediate < latest).all(x => none of x.write.paths are in latest.writes.paths and none of x.read.paths are in latest.writes.paths)
136
+ // - candidate.writes.paths is a subset of latest.writes.paths
137
+ // - test replace writes of x with previous values of x, as time of x
138
+ // rerun latest
139
+ // verify writes2.writes === writes.writes (paths and values)
140
+ // NOTE: See "delayCommit" in syncSchema for the risks of delaying commits.
141
+
142
+ // Find the first write that is a candidate (due to our simplified condition this will only be 1).
143
+ let curIndex = index - 1;
144
+ while (curIndex >= 0) {
145
+ let candidatePred = callPending[curIndex].result;
146
+ if (!candidatePred) {
147
+ // NOTE: This shouldn't happen, but if it does... we could just stop searching here, and not
148
+ // remove any predictions.
149
+ throw new Error("Predictions finished out of order.");
150
+ }
151
+ if (candidateCouldBeChangedByWritesSimilarToOurs(candidatePred)) {
152
+ break;
153
+ }
154
+ curIndex--;
155
+ }
156
+ // Nothing to remove, we don't overlap any other writes
157
+ if (curIndex < 0) {
158
+ return;
159
+ }
160
+
161
+ let candidate = callPending[curIndex];
162
+ let candidatePred = candidate.result;
163
+ if (!candidatePred) {
164
+ throw new Error("Predictions finished out of order.");
165
+ }
166
+ // candidate writes have to be a strict subset of our writes, otherwise there is no way we will clobber the candidate.
167
+ if (candidatePred.writes.some(write => !ourWritePaths.has(write.path))) return;
168
+
169
+
170
+ let debugName = `[redundant check]|${call.DomainName}.${call.FunctionId}`;
171
+ // NOTE: Run right before the call, so we skip it's writes, but use everything else. We could also use
172
+ // candidate.time + epsilon, because none of the values between call and candidate matter (they don't write
173
+ // to any paths either read from).
174
+ let beforeCall = { ...call };
175
+ beforeCall.runAtTime = timeMinusEpsilon(getPredictTime(call.runAtTime));
176
+ let withoutCandidateResult = await getCallWrites({
177
+ call: beforeCall,
178
+ debugName,
179
+ overrides: candidatePred.replacedWriteValues
180
+ });
181
+
182
+ function areWritesEqual(writes: PathValue[], writes2: PathValue[]) {
183
+ if (writes.length !== writes2.length) return false;
184
+ // Write order should probably be the same anyways, so... it is far to compare in order
185
+ for (let i = 0; i < writes.length; i++) {
186
+ let write = writes[i];
187
+ let write2 = writes2[i];
188
+ if (write.path !== write2.path) return false;
189
+ if (pathValueSerializer.compareValuePaths(write, write2) !== 0) {
190
+ return false;
191
+ }
192
+ }
193
+ return true;
194
+ }
195
+
196
+ if (!areWritesEqual(withoutCandidateResult.writes, prediction.writes)) {
197
+ if (Querysub.DEBUG_CALLS) {
198
+ // Failed to collapse delayed functions, because one of the previous predictions was used in the new predicted function. This is valid, but slower than necessary. If pure atomic behavior is not required (which it presumably isn't required, as delayCommit is already being used), try to rewrite the function to be in the form of a set, instead of an add. For example, "function add(count) { x += count; }" => "function set(value) { x = value; }" (calling with set(x + count)).
199
+ console.log(`[Querysub] ${yellow("CANNOT DELAY COLLAPSE")} ${debugCallSpec(candidate.call)} `);
200
+ }
201
+ return;
202
+ }
203
+
204
+ let index2 = callPending.indexOf(candidate);
205
+ if (index2 < 0) return;
206
+ // At this point... we KNOW:
207
+ // a) There are no writes between us and `candidate` (writes are never inserted in the middle, only removed,
208
+ // so this condition will hold despite our delay)
209
+ // b) `candidate` is not required for `call` to run
210
+ // c) `call` replaces all values inside of `candidate`
211
+ // Which means, `candidate is entirely redunant, and can be removed!
212
+
213
+ if (Querysub.DEBUG_CALLS) {
214
+ console.log(`[Querysub] ${magenta("removed redundant call")} ${green(debugCallSpec(candidate.call))} @${debugTime(candidate.call.runAtTime)}`);
215
+ }
216
+ callPending.splice(index2, 1);
217
+ // Cancel as well, in case it isn't redundant, so we can see the issue immediately.
218
+ candidate.cancel();
219
+
220
+ // Move our time back, otherwise repeated delays can result in not committing writes for
221
+ // a long time (indefinitely really), which can result in a lot of lost work, and
222
+ // has little speed benefit (comitting a write every 5 seconds is fine).
223
+ delayedCall.time = candidate.time;
224
+ }));
225
+
226
+ if (Querysub.DEBUG_CALLS) {
227
+ console.log(`[Querysub] ${magenta("delaying")} ${green(debugCallSpec(call))} @${debugTime(call.runAtTime)}`);
228
+ }
229
+ return;
230
+ }
231
+ }
232
+
233
+ await flushDelayedFunctions();
234
+ // NOTE: NO MORE calls can be queued here, because addCall is run in serial, so... we don't need
235
+ // to worry about new calls with times > call.runAtTime (and if it happens we'll get a fairly
236
+ // clear error clientside, which will show up in a notification, so it is a very safe assumption).
237
+ await baseAddCall(call, nodeId, cancel, "call");
238
+ });
239
+
240
+ function getPredictTime(time: Time) {
241
+ return { time: time.time, version: -2, creatorId: time.creatorId };
242
+ }
243
+ const predictionLockVersion = -2344282313;
244
+ export function isOurPrediction(value: PathValue) {
245
+ return value.locks.length === 1 && value.locks[0].startTime.version === predictionLockVersion;
246
+ }
247
+ registerIsOurPrediction(isOurPrediction);
248
+
249
+ interface PredictResult {
250
+ readPaths: Set<string>;
251
+ readParentPaths: Set<string>;
252
+ writes: PathValue[];
253
+ /** Writes that have been replaced AT the write time of the prediction */
254
+ replacedWriteValues: PathValue[];
255
+ }
256
+ function predictCall(config: {
257
+ call: CallSpec;
258
+ localOnly?: boolean;
259
+ overrides?: PathValue[];
260
+ }): {
261
+ cancel: () => void;
262
+ predictPromise: Promise<PredictResult>;
263
+ } {
264
+ let call = config.call;
265
+ let pathResultWrite = getCallResultPath(call);
266
+ Benchmark.onStartPredictCall(pathResultWrite);
267
+ let functionResult: FunctionResult = {
268
+ // Special prediction values (we don't even really need to put anything here, but it
269
+ // might as well have the right format, in case a bug causes it to be used somewhere)
270
+ runCount: -1,
271
+ timeTaken: -1,
272
+ evalTime: 0,
273
+ };
274
+
275
+ // NOTE: We don't cascade predictions, instead only depending on the call succeeding. This is because if we depend on writes,
276
+ // we will be invalidated before we get the actual call result, which results in going back in time, a delay, then
277
+ // having the actual call results, and so jumping forward again. It is better to just wait, so our latest value
278
+ // becomes the latest call value, when we receive it.
279
+ let predictResultWrite: PathValue = {
280
+ path: pathResultWrite,
281
+ value: functionResult,
282
+ locks: [
283
+ {
284
+ // We use a special version, so we don't have to worry about this actually existing. We don't really want
285
+ // a lock on startTime, we just want the range lock.
286
+ startTime: { time: call.runAtTime.time, version: predictionLockVersion, creatorId: call.runAtTime.creatorId },
287
+ // NOTE: The version is special, and used to detect prediction rejections in some logging.
288
+ // NOTE: We need to set our end time enough in the future so errors cause us to be rejected
289
+ endTime: { time: call.runAtTime.time + MAX_CHANGE_AGE, version: Number.MAX_SAFE_INTEGER, creatorId: call.runAtTime.creatorId },
290
+ path: pathResultWrite,
291
+ // Technically we are depending on a future time, so... we are depending on ourself? And our value isn't undefined.
292
+ readIsTranparent: false,
293
+ keepRejectedUntil: call.runAtTime.time + CLIENTSIDE_PREDICT_LEEWAY
294
+ }
295
+ ],
296
+ lockCount: 1,
297
+ // We right our result BEFORE the actual result, so it will be clobbered (and before our read lock!)
298
+ // when the result arrives (although... it will ALSO be rejected, so... this might be less important...)
299
+ // (Also, more importantly, this ha to be UNIQUE, otherwise time locks break!)
300
+ time: getPredictTime(call.runAtTime),
301
+ valid: true,
302
+ event: true,
303
+ };
304
+ let predictLocks: ReadLock[] = [
305
+ {
306
+ startTime: predictResultWrite.time,
307
+ endTime: predictResultWrite.time,
308
+ path: pathResultWrite,
309
+ readIsTranparent: false,
310
+ }
311
+ ];
312
+ let predictions: PredictResult | undefined;
313
+ let predictPromise = predictRunCommitLoop(async () => {
314
+ if (Querysub.DEBUG_PREDICTIONS) {
315
+ console.log(magenta(`Start predict call`), `${call.DomainName}.${call.FunctionId}`);
316
+ }
317
+
318
+ if (!config.localOnly) {
319
+ // Watch the result, so we know when our prediction is rejected (which will be as soon as the result
320
+ // has a real value).
321
+ remoteWatcher.watchLatest({ paths: [pathResultWrite], parentPaths: [] });
322
+ }
323
+ let debugName = `[predict]|${call.DomainName}.${call.FunctionId}`;
324
+
325
+ let dryRunResult: {
326
+ writes: PathValue[];
327
+ readPaths: Set<string>;
328
+ readParentPaths: Set<string>;
329
+ };
330
+ try {
331
+ dryRunResult = await getCallWrites({ call, debugName, overrides: config.overrides });
332
+ } catch (e) {
333
+
334
+ if (!pathValueSerializer.getPathValue(authorityStorage.getValueAtTime(pathResultWrite, undefined))) {
335
+ console.log(`Skipping prediction for ${debugName} due to error running predictive call. Likely just an out of order error.`, e);
336
+ } else {
337
+ // NOTE: This case happens a lot, because of how we handle locks. We don't receive locked values, and so
338
+ // we assume all values have no locks, and only keep the latest. This is usually fine, but... if we lose
339
+ // the race to predict the function against the server updating it, it is likely our prediction will now
340
+ // be running before the latest write. In which case (as we don't really store write history), we will read undefined.
341
+ // This isn't accurate, but... our write WILL almost certainly be wrong (as the value changed), so we don't log here.
342
+ }
343
+ return {
344
+ writes: [],
345
+ readPaths: new Set(),
346
+ readParentPaths: new Set(),
347
+ replacedWriteValues: [],
348
+ };
349
+ }
350
+ predictions = {
351
+ writes: dryRunResult.writes,
352
+ readPaths: dryRunResult.readPaths,
353
+ readParentPaths: dryRunResult.readParentPaths,
354
+ replacedWriteValues: dryRunResult.writes.map(write => {
355
+ let path = write.path;
356
+ let newWrite: PathValue = authorityStorage.getValueAtTime(write.path, write.time) || {
357
+ path, valid: true, time: write.time, locks: [], lockCount: 0, value: undefined, canGCValue: true, isTransparent: true,
358
+ };
359
+ newWrite = { ...newWrite };
360
+ // Use a time very slightly after, so it clobbers the write, and any writes before it, but none after.
361
+ newWrite.time = write.time;
362
+ write.time = {
363
+ time: write.time.time,
364
+ version: write.time.version,
365
+ creatorId: write.time.creatorId + 1,
366
+ };
367
+ return newWrite;
368
+ }),
369
+ };
370
+
371
+ for (let value of predictions.writes) {
372
+ value.source = debugName;
373
+ }
374
+
375
+ // If we already received the actual call, we can't add the prediction, as... the invalidation
376
+ // code won't properly immediately reject our prediction, as we are not the authority on the
377
+ // path, so it treats it as source of truth.
378
+ if (authorityStorage.getValueAtTime(pathResultWrite, undefined)?.value) {
379
+ if (Querysub.DEBUG_PREDICTIONS) {
380
+ console.log(magenta(`Abort predict call, already received call result`), `${call.DomainName}.${call.FunctionId}`);
381
+ }
382
+ return predictions;
383
+ }
384
+
385
+ // We commit slightly different writes, for lock invalidation. Our caller doesn't need / doesn't want this
386
+ // extra write, so we just don't return it.
387
+ // NOTE: I guess we technically don't need to actually write to the result, but... might as well?
388
+ predictions.writes.push(predictResultWrite);
389
+ // Update all locks to use the same locks as predictResultWrite
390
+ // Ensure we use the predict time, otherwise these times will be reused when the call actually happens,
391
+ // potentially with different values, which breaks all of our ReadLocks!
392
+ for (let predict of predictions.writes) {
393
+ predict.time = predictResultWrite.time;
394
+ predict.locks = predictLocks;
395
+ predict.lockCount = predictLocks.length;
396
+ }
397
+
398
+ pathValueCommitter.ingestValues(predictions.writes, undefined, undefined);
399
+
400
+ // NOTE: We have to forcefully setup the lock watches, as we are no the authority on these paths
401
+ // so it is odd for us to do this! It works out though, because the path is unique enough
402
+ // that we don't have to worry about any other valid states coming in for our lock.
403
+ // - Also, once we set up this lock, we should get full reject cascade support.
404
+ lockToCallback.watchLock(predictResultWrite.locks[0], [predictResultWrite]);
405
+ lockToCallback.watchLock(predictLocks[0], predictions.writes);
406
+
407
+ if (Querysub.DEBUG_PREDICTIONS) {
408
+ console.log(magenta(`Finished predict call`), `${call.DomainName}.${call.FunctionId}`);
409
+ }
410
+ return predictions;
411
+ });
412
+ logErrors(predictPromise);
413
+
414
+ let didCancel = false;
415
+ // ALWAYS reject the prediction eventually, in case the function runner server is down.
416
+ // - ALSO, once it goes back up, the write will be too far in the future, and our lock won't be rejected.
417
+ // And don't really want endTime to be infinitely in the future, as then cleanup code has a harder time
418
+ // removing the lock, as then in theory it can always be rejected, no matter how long we wait.
419
+ setTimeout(rejectPrediction, Querysub.PREDICTION_MAX_LIFESPAN);
420
+ function rejectPrediction() {
421
+ if (didCancel) return;
422
+ didCancel = true;
423
+ // Reject our predictions, as the call likely never got committed, so it will never be written
424
+ pathValueCommitter.ingestValidStates([{
425
+ path: pathResultWrite,
426
+ isValid: false,
427
+ time: predictResultWrite.locks[0].startTime,
428
+ isTransparent: false,
429
+ }]);
430
+ }
431
+ setTimeout(auditPrediction, Querysub.PREDICTION_MAX_LIFESPAN);
432
+ function auditPrediction() {
433
+ if (didCancel) return;
434
+ if (Querysub.AUDIT_PREDICTIONS && predictions) {
435
+ const afterTime = { time: call.runAtTime.time, version: Number.MAX_SAFE_INTEGER, creatorId: 0 };
436
+ // Clone predictions, to strip symbols
437
+ let predictionsCopy = cborEncoder().decode(cborEncoder().encode(predictions)) as typeof predictions;
438
+ for (let predict of predictionsCopy.writes ?? []) {
439
+ if (predict.path === pathResultWrite) continue;
440
+ let finalValueObj = authorityStorage.getValueAtTime(predict.path, afterTime);
441
+ if (!authorityStorage.isSynced(predict.path)) continue;
442
+
443
+ let finalValue = pathValueSerializer.getPathValue(finalValueObj);
444
+ let predictValue = pathValueSerializer.getPathValue(predict);
445
+
446
+ let finalBuffer = cborEncoder().encode(finalValue);
447
+ let predictBuffer = cborEncoder().encode(predictValue);
448
+ // If they are different, warn
449
+ if (!finalBuffer.equals(predictBuffer)) {
450
+ // NOTE: This MIGHT be due to delayCommit on another function. We can't know when to force flush
451
+ // delayCommit values, and so we only force flush if other (non-delayCommit) functions are committed
452
+ // in the same schema, which will be wrong if data is accessed cross schema.
453
+ // - If this happens, and is a problem, we COULD handle it, but... it is very difficult, as we would need
454
+ // to evaluate the other function, detect it conflicts with a delayCommit functions, then run the delayCommit
455
+ // function, then re-evaluated the conflicted prediction (which is a lot of communication and reruns
456
+ // between a lot of systems... especially because in a second this will resolve by the server
457
+ // clobbering our prediction).
458
+ // - AND, we can't commit the pending calls until we sort out the order, so this necessarily requires
459
+ // slowing down commits if we are delaying other calls.
460
+ console.warn(`${red("Prediction was wrong")}: ${getPathFromStr(predict.path).join(".")} predict != finalValue`, predictValue, finalValue);
461
+ }
462
+ }
463
+ }
464
+ }
465
+
466
+ return {
467
+ cancel: rejectPrediction,
468
+ predictPromise,
469
+ };
470
+ }
471
+
472
+ export async function getCallWrites(config: {
473
+ debugName: string;
474
+ call: CallSpec;
475
+ overrides?: PathValue[];
476
+ }) {
477
+ let { call, debugName } = config;
478
+ let { functionSpec } = await proxyWatcher.commitFunction({
479
+ watchFunction: function getModuleConfig() {
480
+ //let moduleConfig = atomicObjectRead(functionSchema()[call.DomainName].PathFunctionRunner[call.ModuleId].Module);
481
+ // if (!(call.DomainName in functionSchema())) {
482
+ // throw new Error(`Domain not found ${call.DomainName}`);
483
+ // }
484
+ let domainObject = functionSchema()[call.DomainName];
485
+ if (!(call.ModuleId in domainObject.PathFunctionRunner)) {
486
+ throw new Error(`Module not found ${call.DomainName}.${call.ModuleId}`);
487
+ }
488
+ let moduleObject = domainObject.PathFunctionRunner[call.ModuleId];
489
+ if (!(call.FunctionId in moduleObject.Sources)) {
490
+ throw new Error(`Function not found ${call.DomainName}.${call.ModuleId}.${call.FunctionId}`);
491
+ }
492
+ let functionSpec = atomicObjectRead(moduleObject.Sources[call.FunctionId]);
493
+ return { functionSpec };
494
+ }
495
+ });
496
+ if (!functionSpec) {
497
+ throw new Error(`Function not found ${call.DomainName}.${call.ModuleId}.${call.FunctionId}`);
498
+ }
499
+
500
+ await addModuleToLoader(functionSpec);
501
+
502
+ let module = await getModuleFromConfig(functionSpec);
503
+ let exportPath = getPathFromStr(functionSpec.exportPathStr);
504
+ let exportObj = module.exports;
505
+ for (let path of exportPath) {
506
+ exportObj = exportObj[path];
507
+ }
508
+ let baseFunction = exportObj as Function;
509
+
510
+ return await proxyWatcher.dryRunFull({
511
+ debugName,
512
+ runAtTime: call.runAtTime,
513
+ overrideAllowLockDomainsPrefixes: [getPathStr1(call.DomainName)],
514
+ // We are going to clobber locks anyways, so there is no reason to create them
515
+ unsafeNoLocks: true,
516
+ overrides: config.overrides,
517
+ nestedCalls: "inline",
518
+ watchFunction() {
519
+ overrideCurrentCall(call, () => {
520
+ let args = parseArgs(call);
521
+ baseFunction(...args);
522
+ });
523
+ },
524
+ });
525
+ }
@@ -0,0 +1,67 @@
1
+ import preact from "preact";
2
+ import { qreact } from "../4-dom/qreact";
3
+ import { Button } from "../library-components/Button";
4
+ import { Modal } from "./Modal";
5
+
6
+
7
+ export class FullscreenModal extends preact.Component<{
8
+ parentState?: { open: boolean };
9
+ onCancel?: () => void;
10
+ style?: preact.JSX.CSSProperties;
11
+ outerStyle?: preact.JSX.CSSProperties;
12
+ }> {
13
+ render() {
14
+ let { parentState } = this.props;
15
+ return (
16
+ <Modal>
17
+ <div style={{ display: "none" }}>
18
+ <Button hotkeys={["Global+Escape"]} onClick={() => {
19
+ if (parentState) parentState.open = false;
20
+ this.props.onCancel?.();
21
+ }}>Close</Button>
22
+ </div>
23
+ <div
24
+ style={{
25
+ position: "fixed",
26
+ top: 0,
27
+ left: 0,
28
+ width: "100vw",
29
+ height: "100vh",
30
+ background: "hsla(0, 0%, 30%, 0.5)",
31
+ padding: 100,
32
+ display: "flex",
33
+ alignItems: "center",
34
+ justifyContent: "center",
35
+ overflow: "auto",
36
+ cursor: "pointer",
37
+ ...this.props.outerStyle,
38
+ }}
39
+ onClick={e => {
40
+ if (qreact.isTarget(e)) {
41
+ if (parentState) parentState.open = false;
42
+ this.props.onCancel?.();
43
+ }
44
+ }}
45
+ >
46
+ <div
47
+ style={{
48
+ background: "hsl(0, 0%, 100%)",
49
+ padding: 20,
50
+ color: "hsl(0, 0%, 7%)",
51
+ cursor: "default",
52
+ width: "100%",
53
+ display: "flex",
54
+ flexDirection: "column",
55
+ gap: 10,
56
+ maxHeight: "calc(100% - 200px)",
57
+ overflow: "auto",
58
+ ...this.props.style
59
+ }}
60
+ >
61
+ {this.props.children}
62
+ </div>
63
+ </div>
64
+ </Modal>
65
+ );
66
+ }
67
+ }