querysub 0.504.0 → 0.506.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/function.js +3 -3
- package/bin/server-dev.js +11 -0
- package/bin/server.js +2 -0
- package/package.json +2 -1
- package/src/0-path-value-core/AuthorityLookup.ts +1 -0
- package/src/0-path-value-core/PathRouter.ts +27 -4
- package/src/0-path-value-core/PathRouterRouteOverride.ts +98 -18
- package/src/0-path-value-core/PathRouterServerAuthoritySpec.tsx +4 -1
- package/src/0-path-value-core/PathValueCommitter.ts +6 -1
- package/src/0-path-value-core/PathValueController.ts +7 -5
- package/src/0-path-value-core/PathWatcher.ts +11 -8
- package/src/0-path-value-core/hackedPackedPathParentFiltering.ts +24 -10
- package/src/0-path-value-core/pathValueCore.ts +15 -7
- package/src/1-path-client/RemoteWatcher.ts +35 -18
- package/src/3-path-functions/PathFunctionHelpers.ts +14 -15
- package/src/3-path-functions/PathFunctionRunner.ts +18 -12
- package/src/3-path-functions/PathFunctionRunnerMain.ts +5 -7
- package/src/3-path-functions/syncSchema.ts +3 -1
- package/src/4-querysub/Querysub.ts +14 -2
- package/src/4-querysub/QuerysubController.ts +13 -6
- package/src/config.ts +39 -3
- package/src/config2.ts +2 -8
- package/src/diagnostics/logs/errorTickets/TicketPage.tsx +4 -3
- package/src/diagnostics/managementPages.tsx +1 -1
- package/src/diagnostics/misc-pages/AuthoritySpecPage.tsx +30 -5
- package/src/misc/filterable.ts +0 -395
|
@@ -20,6 +20,8 @@ import { auditLog, isDebugLogEnabled } from "../0-path-value-core/auditLogs";
|
|
|
20
20
|
import { debugNodeThread } from "../-c-identity/IdentityController";
|
|
21
21
|
import { authorityLookup } from "../0-path-value-core/AuthorityLookup";
|
|
22
22
|
import { decodeParentFilter, encodeParentFilter, registerGetSpecForChildPath } from "../0-path-value-core/hackedPackedPathParentFiltering";
|
|
23
|
+
import { getPathNetwork } from "../0-path-value-core/PathRouterRouteOverride";
|
|
24
|
+
import { DEFAULT_NETWORK } from "../config";
|
|
23
25
|
import { removeRange, rangesOverlap } from "../rangeMath";
|
|
24
26
|
import { evaluateValidStates } from "../config2";
|
|
25
27
|
|
|
@@ -46,8 +48,10 @@ export class RemoteWatcher {
|
|
|
46
48
|
start: number;
|
|
47
49
|
// exclusive
|
|
48
50
|
end: number;
|
|
49
|
-
// encodeParentFilter({ path: cleanPath, startFraction: start, endFraction: end })
|
|
51
|
+
// encodeParentFilter({ path: cleanPath, startFraction: start, endFraction: end, network })
|
|
50
52
|
finalPath: string;
|
|
53
|
+
// undefined means the default network
|
|
54
|
+
network?: string;
|
|
51
55
|
// All of the requests paths that request it (pre decodeParentFilter).
|
|
52
56
|
inputPaths: Set<string>;
|
|
53
57
|
|
|
@@ -139,8 +143,10 @@ export class RemoteWatcher {
|
|
|
139
143
|
let parentPath = getParentPathStr(path);
|
|
140
144
|
let watchObj = this.remoteWatchParents2.get(parentPath);
|
|
141
145
|
if (!watchObj) return undefined;
|
|
142
|
-
|
|
146
|
+
let pathNetwork = getPathNetwork(path);
|
|
147
|
+
// NOTE: PathRouter.getChildReadNodes PROMISES That all of the nodes will hash in the same way, and if they all hash in the same way, and they have different ranges, then every path will uniquely map to a single value.
|
|
143
148
|
return watchObj.ranges.find(x => {
|
|
149
|
+
if ((x.network || DEFAULT_NETWORK) !== pathNetwork) return false;
|
|
144
150
|
let route = PathRouter.getRouteFull({ path, spec: x.authoritySpec });
|
|
145
151
|
return x.start <= route && route < x.end;
|
|
146
152
|
});
|
|
@@ -149,8 +155,10 @@ export class RemoteWatcher {
|
|
|
149
155
|
let parentPath = getParentPathStr(path);
|
|
150
156
|
let watchObj = this.remoteWatchParents2.get(parentPath);
|
|
151
157
|
if (!watchObj) return undefined;
|
|
152
|
-
|
|
158
|
+
let pathNetwork = getPathNetwork(path);
|
|
159
|
+
// NOTE: PathRouter.getChildReadNodes PROMISES That all of the nodes will hash in the same way, and if they all hash in the same way, and they have different ranges, then every path will uniquely map to a single value.
|
|
153
160
|
for (let range of watchObj.ranges) {
|
|
161
|
+
if ((range.network || DEFAULT_NETWORK) !== pathNetwork) continue;
|
|
154
162
|
let route = PathRouter.getRouteFull({ path, spec: range.authoritySpec });
|
|
155
163
|
if (range.start <= route && route < range.end) {
|
|
156
164
|
return route;
|
|
@@ -258,17 +266,17 @@ export class RemoteWatcher {
|
|
|
258
266
|
if (totalMissingPaths === 0) {
|
|
259
267
|
let authorities = authorityLookup.getTopologySync();
|
|
260
268
|
authorityId = PathRouter.getReadyAuthority(path)?.nodeId;
|
|
261
|
-
console.warn(`Missing authority for path ${path}`, { authorities });
|
|
269
|
+
console.warn(`Missing authority for path ${path} (network ${getPathNetwork(path)})`, { network: getPathNetwork(path), authorities });
|
|
262
270
|
}
|
|
263
271
|
totalMissingPaths++;
|
|
264
272
|
if (!this.disconnectedPaths.has(path)) {
|
|
265
273
|
newDisconnectPaths++;
|
|
266
274
|
this.disconnectedPaths.add(path);
|
|
267
|
-
auditLog("NO AUTHORITY FOR PATH", { path });
|
|
275
|
+
auditLog("NO AUTHORITY FOR PATH", { path, network: getPathNetwork(path) });
|
|
268
276
|
}
|
|
269
277
|
continue;
|
|
270
278
|
}
|
|
271
|
-
auditLog("FOUND AUTHORITY FOR PATH", { path, authorityId });
|
|
279
|
+
auditLog("FOUND AUTHORITY FOR PATH", { path, network: getPathNetwork(path), authorityId });
|
|
272
280
|
this.disconnectedPaths.delete(path);
|
|
273
281
|
foundPaths++;
|
|
274
282
|
|
|
@@ -289,6 +297,7 @@ export class RemoteWatcher {
|
|
|
289
297
|
|
|
290
298
|
let decodedParentFilter = decodeParentFilter(path);
|
|
291
299
|
let cleanPath = decodedParentFilter?.path ?? path;
|
|
300
|
+
let network = decodedParentFilter?.network;
|
|
292
301
|
|
|
293
302
|
let rangeStart = 0;
|
|
294
303
|
let rangeEnd = 1;
|
|
@@ -297,6 +306,8 @@ export class RemoteWatcher {
|
|
|
297
306
|
rangeEnd = decodedParentFilter.end;
|
|
298
307
|
}
|
|
299
308
|
|
|
309
|
+
const sameNetwork = (rangeNetwork: string | undefined) => (rangeNetwork || DEFAULT_NETWORK) === (network || DEFAULT_NETWORK);
|
|
310
|
+
|
|
300
311
|
let watchObj = this.remoteWatchParents2.get(cleanPath);
|
|
301
312
|
|
|
302
313
|
if (watchObj && !config.tryReconnect) {
|
|
@@ -306,11 +317,13 @@ export class RemoteWatcher {
|
|
|
306
317
|
end: rangeEnd,
|
|
307
318
|
}];
|
|
308
319
|
for (let range of watchObj.ranges) {
|
|
320
|
+
if (!sameNetwork(range.network)) continue;
|
|
309
321
|
removeRange(requiredRanges, range);
|
|
310
322
|
}
|
|
311
323
|
if (requiredRanges.length === 0) {
|
|
312
324
|
this.disconnectedParents.delete(path);
|
|
313
325
|
for (let range of watchObj.ranges) {
|
|
326
|
+
if (!sameNetwork(range.network)) continue;
|
|
314
327
|
if (rangesOverlap(range, { start: rangeStart, end: rangeEnd })) {
|
|
315
328
|
range.inputPaths.add(path);
|
|
316
329
|
}
|
|
@@ -319,7 +332,8 @@ export class RemoteWatcher {
|
|
|
319
332
|
}
|
|
320
333
|
}
|
|
321
334
|
|
|
322
|
-
let
|
|
335
|
+
let readNodesPath = network && encodeParentFilter({ path: cleanPath, startFraction: 0, endFraction: 1, network }) || cleanPath;
|
|
336
|
+
let { nodes } = PathRouter.getChildReadNodes(readNodesPath, {
|
|
323
337
|
// Pass existing connected node ids as preferred
|
|
324
338
|
preferredNodeIds: watchObj?.ranges.map(x => x.authorityId) || []
|
|
325
339
|
});
|
|
@@ -347,6 +361,7 @@ export class RemoteWatcher {
|
|
|
347
361
|
// Compute which parts of [rangeStart, rangeEnd] aren't yet covered by existing watches
|
|
348
362
|
let missingRanges: { start: number; end: number }[] = [{ start: rangeStart, end: rangeEnd }];
|
|
349
363
|
for (let range of watchObj.ranges) {
|
|
364
|
+
if (!sameNetwork(range.network)) continue;
|
|
350
365
|
removeRange(missingRanges, range);
|
|
351
366
|
}
|
|
352
367
|
|
|
@@ -359,13 +374,14 @@ export class RemoteWatcher {
|
|
|
359
374
|
let clippedStart = Math.max(node.range.start, missing.start);
|
|
360
375
|
let clippedEnd = Math.min(node.range.end, missing.end);
|
|
361
376
|
|
|
362
|
-
let finalPath = encodeParentFilter({ path: cleanPath, startFraction: clippedStart, endFraction: clippedEnd });
|
|
377
|
+
let finalPath = encodeParentFilter({ path: cleanPath, startFraction: clippedStart, endFraction: clippedEnd, network });
|
|
363
378
|
let existingRange = watchObj.ranges.find(x => x.finalPath === finalPath && x.authorityId === node.nodeId);
|
|
364
379
|
if (!existingRange) {
|
|
365
380
|
existingRange = {
|
|
366
381
|
start: clippedStart,
|
|
367
382
|
end: clippedEnd,
|
|
368
383
|
finalPath,
|
|
384
|
+
network,
|
|
369
385
|
inputPaths: new Set(),
|
|
370
386
|
authorityId: node.nodeId,
|
|
371
387
|
suppressedWatches: new Set(),
|
|
@@ -385,6 +401,7 @@ export class RemoteWatcher {
|
|
|
385
401
|
|
|
386
402
|
// Register path on pre-existing ranges that already cover parts of [rangeStart, rangeEnd]
|
|
387
403
|
for (let range of watchObj.ranges) {
|
|
404
|
+
if (!sameNetwork(range.network)) continue;
|
|
388
405
|
if (rangesOverlap(range, { start: rangeStart, end: rangeEnd })) {
|
|
389
406
|
range.inputPaths.add(path);
|
|
390
407
|
}
|
|
@@ -396,7 +413,7 @@ export class RemoteWatcher {
|
|
|
396
413
|
console.warn(`Some paths have no authority. We will search for an authority periodically until we find an authority. New missing ${newDisconnectPaths} paths and ${newDisconnectParents} parent paths, total missing ${this.disconnectedPaths.size} paths and ${this.disconnectedParents.size} parent paths.`);
|
|
397
414
|
let first10 = Array.from(this.disconnectedPaths).slice(0, 10);
|
|
398
415
|
for (let path of first10) {
|
|
399
|
-
console.log(`\t${path}`);
|
|
416
|
+
console.log(`\t(network ${getPathNetwork(path)}) ${path}`);
|
|
400
417
|
}
|
|
401
418
|
if (this.disconnectedPaths.size > 10) {
|
|
402
419
|
console.log(`\t... and ${this.disconnectedPaths.size - 10} more paths`);
|
|
@@ -416,10 +433,10 @@ export class RemoteWatcher {
|
|
|
416
433
|
if (isOwnNodeId(authorityId)) continue;
|
|
417
434
|
|
|
418
435
|
for (let path of paths) {
|
|
419
|
-
auditLog("remoteWatcher outer WATCH", { path, remoteNodeId: authorityId });
|
|
436
|
+
auditLog("remoteWatcher outer WATCH", { path, network: getPathNetwork(path), remoteNodeId: authorityId });
|
|
420
437
|
}
|
|
421
438
|
for (let path of parentPaths) {
|
|
422
|
-
auditLog("remoteWatcher outer PARENT WATCH", { path, remoteNodeId: authorityId });
|
|
439
|
+
auditLog("remoteWatcher outer PARENT WATCH", { path, network: decodeParentFilter(path)?.network || DEFAULT_NETWORK, remoteNodeId: authorityId });
|
|
423
440
|
}
|
|
424
441
|
|
|
425
442
|
if (STOP_KEYS_DOUBLE_SENDS) {
|
|
@@ -428,7 +445,7 @@ export class RemoteWatcher {
|
|
|
428
445
|
let range = this.getRemoteWatchParentRange(path);
|
|
429
446
|
if (range) {
|
|
430
447
|
range.suppressedWatches.add(path);
|
|
431
|
-
auditLog("remoteWatcher outer WATCH SUPPRESSED", { path, remoteNodeId: authorityId });
|
|
448
|
+
auditLog("remoteWatcher outer WATCH SUPPRESSED", { path, network: getPathNetwork(path), remoteNodeId: authorityId });
|
|
432
449
|
return false;
|
|
433
450
|
}
|
|
434
451
|
return true;
|
|
@@ -480,10 +497,10 @@ export class RemoteWatcher {
|
|
|
480
497
|
}
|
|
481
498
|
for (let [authorityId, { paths, parentPaths }] of byAuthority) {
|
|
482
499
|
for (let path of paths) {
|
|
483
|
-
auditLog("remoteWatcher inner WATCH", { path, remoteNodeId: authorityId });
|
|
500
|
+
auditLog("remoteWatcher inner WATCH", { path, network: getPathNetwork(path), remoteNodeId: authorityId });
|
|
484
501
|
}
|
|
485
502
|
for (let path of parentPaths) {
|
|
486
|
-
auditLog("remoteWatcher inner WATCH PARENT", { path, remoteNodeId: authorityId });
|
|
503
|
+
auditLog("remoteWatcher inner WATCH PARENT", { path, network: decodeParentFilter(path)?.network || DEFAULT_NETWORK, remoteNodeId: authorityId });
|
|
487
504
|
}
|
|
488
505
|
// NOTE: We log watches here because we want to log batched watches
|
|
489
506
|
ActionsHistory.OnNewWatches({ newPathsWatched: paths, newParentsWatched: parentPaths, debugName: batched[0].debugName, authorityId });
|
|
@@ -494,10 +511,10 @@ export class RemoteWatcher {
|
|
|
494
511
|
};
|
|
495
512
|
if (isDebugLogEnabled()) {
|
|
496
513
|
for (let path of paths) {
|
|
497
|
-
auditLog("Asking to watch path", { path, authorityId, targetNodeThreadId: debugNodeThread(authorityId) });
|
|
514
|
+
auditLog("Asking to watch path", { path, network: getPathNetwork(path), authorityId, targetNodeThreadId: debugNodeThread(authorityId) });
|
|
498
515
|
}
|
|
499
516
|
for (let path of parentPaths) {
|
|
500
|
-
auditLog("Asking to watch parent path", { path, authorityId, targetNodeThreadId: debugNodeThread(authorityId) });
|
|
517
|
+
auditLog("Asking to watch parent path", { path, network: decodeParentFilter(path)?.network || DEFAULT_NETWORK, authorityId, targetNodeThreadId: debugNodeThread(authorityId) });
|
|
501
518
|
}
|
|
502
519
|
}
|
|
503
520
|
logErrors(RemoteWatcher.REMOTE_WATCH_FUNCTION(config, authorityId));
|
|
@@ -587,10 +604,10 @@ export class RemoteWatcher {
|
|
|
587
604
|
}
|
|
588
605
|
for (let [authorityIdBase, { paths, parentPaths }] of unwatchesPerAuthority.entries()) {
|
|
589
606
|
for (let path of paths) {
|
|
590
|
-
auditLog("remoteWatcher inner UNWATCH", { path, remoteNodeId: authorityIdBase });
|
|
607
|
+
auditLog("remoteWatcher inner UNWATCH", { path, network: getPathNetwork(path), remoteNodeId: authorityIdBase });
|
|
591
608
|
}
|
|
592
609
|
for (let path of parentPaths) {
|
|
593
|
-
auditLog("remoteWatcher inner UNWATCH PARENT", { path, remoteNodeId: authorityIdBase });
|
|
610
|
+
auditLog("remoteWatcher inner UNWATCH PARENT", { path, network: decodeParentFilter(path)?.network || DEFAULT_NETWORK, remoteNodeId: authorityIdBase });
|
|
594
611
|
}
|
|
595
612
|
if (!isOwnNodeId(authorityIdBase)) {
|
|
596
613
|
logErrors(RemoteWatcher.REMOTE_UNWATCH_FUNCTION({ paths, parentPaths }, authorityIdBase));
|
|
@@ -14,13 +14,12 @@ import { blue, green, red } from "socket-function/src/formatting/logColors";
|
|
|
14
14
|
import { getPathStr2 } from "../path";
|
|
15
15
|
import { isNode, sort } from "socket-function/src/misc";
|
|
16
16
|
import { decodeCborx, encodeCborx } from "../misc/cloneHelpers";
|
|
17
|
-
import { parseFilterable } from "../misc/filterable";
|
|
18
17
|
import { interceptCalls } from "../-0-hooks/hooks";
|
|
19
18
|
import { createRoutingOverrideKey } from "../0-path-value-core/PathRouterRouteOverride";
|
|
20
19
|
import { PathRouter } from "../0-path-value-core/PathRouter";
|
|
21
20
|
import { getPathFromProxy } from "../2-proxy/pathValueProxy";
|
|
22
|
-
import { getDomain, isPublic } from "../config";
|
|
23
|
-
import {
|
|
21
|
+
import { getDomain, getPrimaryNetwork, isPublic, DEFAULT_NETWORK } from "../config";
|
|
22
|
+
import { isServer } from "../config2";
|
|
24
23
|
import { getTimeUnique } from "socket-function/src/bits";
|
|
25
24
|
|
|
26
25
|
// NOTE: We could deploy single functions, but... we will almost always be updating all functions at
|
|
@@ -161,16 +160,27 @@ export function writeFunctionCall(config: {
|
|
|
161
160
|
// which gives them about a minute to guess it...
|
|
162
161
|
let callId = now + "_" + secureRandom();
|
|
163
162
|
|
|
163
|
+
// Clients don't know their network. The QuerysubController decides the network for their calls (and rewrites the callId to match).
|
|
164
|
+
let network: string | undefined;
|
|
165
|
+
if (isNode()) {
|
|
166
|
+
network = getPrimaryNetwork();
|
|
167
|
+
if (network === DEFAULT_NETWORK) {
|
|
168
|
+
network = undefined;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
164
172
|
// We HAVE to override the routing id, otherwise PathFunctionRunner secondary sharding won't work
|
|
165
173
|
callId = getCallIdOverride({
|
|
166
174
|
moduleId,
|
|
167
175
|
functionId,
|
|
168
176
|
callId,
|
|
169
177
|
args,
|
|
178
|
+
network,
|
|
170
179
|
}) || createRoutingOverrideKey({
|
|
171
180
|
originalKey: callId,
|
|
172
181
|
routeKey: callId,
|
|
173
182
|
remappedPrefix: getFunctionRunnerPrefix(moduleId),
|
|
183
|
+
network,
|
|
174
184
|
});
|
|
175
185
|
|
|
176
186
|
let argsEncoded = encodeArgs(args);
|
|
@@ -185,19 +195,8 @@ export function writeFunctionCall(config: {
|
|
|
185
195
|
// Will be updated by Querysub to be correct (unless we are directly writing, then...
|
|
186
196
|
// this is fine).
|
|
187
197
|
callerIP: "127.0.0.1",
|
|
198
|
+
network,
|
|
188
199
|
};
|
|
189
|
-
// NOTE: I don't know when this would ever really work. I guess to isolate function calls, but if we're running locally, isn't that easy?
|
|
190
|
-
// if (!isNode()) {
|
|
191
|
-
// // Get the "setfncfilter" querystring parameter
|
|
192
|
-
// let url = new URL(window.location.href);
|
|
193
|
-
// let setfncfilter = url.searchParams.get("setfncfilter");
|
|
194
|
-
// if (setfncfilter) {
|
|
195
|
-
// callSpec.filterable = parseFilterable(setfncfilter);
|
|
196
|
-
// }
|
|
197
|
-
// }
|
|
198
|
-
if (isNode()) {
|
|
199
|
-
callSpec.filterable = getFncFilter();
|
|
200
|
-
}
|
|
201
200
|
|
|
202
201
|
if (curInterceptor) {
|
|
203
202
|
curInterceptor.onCall(callSpec, metadata);
|
|
@@ -16,10 +16,9 @@ import { parseArgs } from "./PathFunctionHelpers";
|
|
|
16
16
|
import { FunctionMetadata, PERMISSIONS_FUNCTION_ID, addRoutingPrefixForDeploy, getAllDevelopmentModulesIds, getDevelopmentModule, getExportPath, getModuleRelativePath, getSchemaObject } from "./syncSchema";
|
|
17
17
|
import { formatTime } from "socket-function/src/formatting/format";
|
|
18
18
|
import { getControllerNodeIdList, set_debug_getFunctionRunnerShards } from "../-g-core-values/NodeCapabilities";
|
|
19
|
-
import { FilterSelector, Filterable, doesMatch } from "../misc/filterable";
|
|
20
19
|
import { SocketFunction } from "socket-function/SocketFunction";
|
|
21
20
|
import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
|
|
22
|
-
import { getDomain, isPublic } from "../config";
|
|
21
|
+
import { getDomain, isPublic, DEFAULT_NETWORK } from "../config";
|
|
23
22
|
import { getGitRefSync, getGitURLSync } from "../4-deploy/git";
|
|
24
23
|
import type { DeployProgress } from "../4-deploy/deployFunctions";
|
|
25
24
|
import { getRoutingOverride, getRoutingOverridePart } from "../0-path-value-core/PathRouterRouteOverride";
|
|
@@ -105,10 +104,11 @@ export interface CallSpec {
|
|
|
105
104
|
callerIP: string;
|
|
106
105
|
runAtTime: Time;
|
|
107
106
|
|
|
108
|
-
// Not just used for debugging, also used to add special proxy-related warnings.
|
|
107
|
+
// Not just used for debugging, also used to add special proxy-related warnings.
|
|
109
108
|
fromProxy?: string;
|
|
110
109
|
|
|
111
|
-
|
|
110
|
+
// The network the call is on ("default" if unset). Only FunctionRunners on this network will run the call.
|
|
111
|
+
network?: string;
|
|
112
112
|
}
|
|
113
113
|
export function debugCallSpec(spec: CallSpec): string {
|
|
114
114
|
return `${spec.DomainName}/${spec.ModuleId}/${spec.FunctionId}`;
|
|
@@ -227,7 +227,8 @@ export class PathFunctionRunner {
|
|
|
227
227
|
shardRange: { startFraction: number, endFraction: number };
|
|
228
228
|
secondaryShardRange?: { startFraction: number, endFraction: number };
|
|
229
229
|
PermissionsChecker: PermissionsCheckType | undefined;
|
|
230
|
-
|
|
230
|
+
// The networks we listen to calls on. Unset is equivalent to ["default"].
|
|
231
|
+
networks?: string[];
|
|
231
232
|
}) {
|
|
232
233
|
SocketFunction.expose(FunctionPreloadController);
|
|
233
234
|
SocketFunction.expose(FunctionCaptureController);
|
|
@@ -264,11 +265,14 @@ export class PathFunctionRunner {
|
|
|
264
265
|
|
|
265
266
|
let outstandingCalls = 0;
|
|
266
267
|
|
|
268
|
+
let networks = this.config.networks && this.config.networks.length > 0 && this.config.networks || [DEFAULT_NETWORK];
|
|
269
|
+
|
|
267
270
|
let watchModuleCalls = cache((moduleId: string) => {
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
271
|
+
for (let network of networks) {
|
|
272
|
+
let networkConfig = network === DEFAULT_NETWORK && fullFraction || { ...fullFraction, network };
|
|
273
|
+
Querysub.keys(functionSchema()[domainName].PathFunctionRunner[moduleId].Calls, networkConfig);
|
|
274
|
+
Querysub.keys(functionSchema()[domainName].PathFunctionRunner[moduleId].Results, networkConfig);
|
|
275
|
+
}
|
|
272
276
|
});
|
|
273
277
|
|
|
274
278
|
let runningCalls = new Set<string>();
|
|
@@ -394,8 +398,8 @@ export class PathFunctionRunner {
|
|
|
394
398
|
}
|
|
395
399
|
|
|
396
400
|
if (PathFunctionRunner.DEBUG_CALLS) {
|
|
397
|
-
// NOTE: If this
|
|
398
|
-
// - The
|
|
401
|
+
// NOTE: If this starts but never runs, it means the call's network didn't match ours, which either means it was a dev call and so not relevant for us, or we are dev and it's a production call, so not relevant for us.
|
|
402
|
+
// - The network is set by --network when running serverside, or, clientside, by --network set on the command line of the querysub server.
|
|
399
403
|
console.log(`QUEUING ${getDebugName(callData, functionSpec, true)}`);
|
|
400
404
|
let resultsPath = getProxyPath(() => moduleData.Results[callId]);
|
|
401
405
|
let history = authorityStorage.getValuePlusHistory(resultsPath);
|
|
@@ -551,7 +555,9 @@ export class PathFunctionRunner {
|
|
|
551
555
|
skipPermissions = PermissionsChecker.skipPermissionsChecks.bind(PermissionsChecker);
|
|
552
556
|
}
|
|
553
557
|
|
|
554
|
-
|
|
558
|
+
let callNetwork = callSpec.network || DEFAULT_NETWORK;
|
|
559
|
+
let ourNetworks = this.config.networks && this.config.networks.length > 0 && this.config.networks || [DEFAULT_NETWORK];
|
|
560
|
+
if (!ourNetworks.includes(callNetwork)) {
|
|
555
561
|
return;
|
|
556
562
|
}
|
|
557
563
|
|
|
@@ -6,7 +6,6 @@ import yargs from "yargs";
|
|
|
6
6
|
let yargObj = yargs(process.argv)
|
|
7
7
|
.option("fncshard", { type: "string", default: "0-1", desc: "Shard range as start-end (e.g., '0-0.5' or '0.25-0.75')" })
|
|
8
8
|
.option("fncsecshard", { type: "string", default: "", desc: "Secondary shard range as start-end (e.g., '0-0.5'). Values in this range (not matched in fncshard) are delayed." })
|
|
9
|
-
.option("filter", { type: "string", default: "", desc: `Filter to only include handle specific function calls (fncshard is still applied). For example, "a&b|c", using regular boolean rules.` })
|
|
10
9
|
.argv
|
|
11
10
|
;
|
|
12
11
|
|
|
@@ -16,9 +15,8 @@ import { SocketFunction } from "socket-function/SocketFunction";
|
|
|
16
15
|
import { getThreadKeyCert } from "sliftutils/misc/https/certs";
|
|
17
16
|
import { ClientWatcher } from "../1-path-client/pathValueClientWatcher";
|
|
18
17
|
import { timeInMinute } from "socket-function/src/misc";
|
|
19
|
-
import { getDomain, isLocal, isPublic } from "../config";
|
|
18
|
+
import { getDomain, getNetworks, isLocal, isPublic, DEFAULT_NETWORK } from "../config";
|
|
20
19
|
import { green, magenta } from "socket-function/src/formatting/logColors";
|
|
21
|
-
import { parseFilterSelector } from "../misc/filterable";
|
|
22
20
|
import path from "path";
|
|
23
21
|
import { IndexedLogs } from "../diagnostics/logs/IndexedLogs/IndexedLogs";
|
|
24
22
|
|
|
@@ -61,7 +59,7 @@ async function main() {
|
|
|
61
59
|
console.log(green(`Sharding from ${shardStart} to ${shardEnd}`));
|
|
62
60
|
}
|
|
63
61
|
|
|
64
|
-
let
|
|
62
|
+
let networks = getNetworks();
|
|
65
63
|
|
|
66
64
|
new PathFunctionRunner({
|
|
67
65
|
domainName: getDomain(),
|
|
@@ -69,7 +67,7 @@ async function main() {
|
|
|
69
67
|
secondaryShardRange,
|
|
70
68
|
// TODO: Maybe abstract this out even more, so anything can plug in permissions checks?
|
|
71
69
|
PermissionsChecker: PermissionsCheck,
|
|
72
|
-
|
|
70
|
+
networks,
|
|
73
71
|
});
|
|
74
72
|
|
|
75
73
|
if (!isPublic()) {
|
|
@@ -78,8 +76,8 @@ async function main() {
|
|
|
78
76
|
await import(deployPath);
|
|
79
77
|
}
|
|
80
78
|
|
|
81
|
-
if (
|
|
82
|
-
console.log(magenta(`Only running functions
|
|
79
|
+
if (networks.length !== 1 || networks[0] !== DEFAULT_NETWORK) {
|
|
80
|
+
console.log(magenta(`Only running functions on the network(s): ${networks.join(", ")}. Use --network ${networks[0]} in your http server to route calls here.`));
|
|
83
81
|
}
|
|
84
82
|
}
|
|
85
83
|
logErrors(main());
|
|
@@ -579,11 +579,12 @@ export function getCallIdOverride(config: {
|
|
|
579
579
|
functionId: string;
|
|
580
580
|
callId: string;
|
|
581
581
|
args: unknown[];
|
|
582
|
+
network?: string;
|
|
582
583
|
}): string {
|
|
583
584
|
try {
|
|
584
585
|
if (querysub) {
|
|
585
586
|
if (!querysub.Querysub.isInSyncedCall()) {
|
|
586
|
-
// NOTE: This is wrong and it will result in loading user being used as a key sometimes. However, it should be fine, as if the call id override is wrong, it shouldn't break anything.
|
|
587
|
+
// NOTE: This is wrong and it will result in loading user being used as a key sometimes. However, it should be fine, as if the call id override is wrong, it shouldn't break anything.
|
|
587
588
|
return querysub.Querysub.localRead(() => getCallIdOverride(config));
|
|
588
589
|
}
|
|
589
590
|
}
|
|
@@ -593,6 +594,7 @@ export function getCallIdOverride(config: {
|
|
|
593
594
|
remappedPrefix: def.prefix,
|
|
594
595
|
originalKey: config.callId,
|
|
595
596
|
routeKey: def.getKey(...config.args),
|
|
597
|
+
network: config.network,
|
|
596
598
|
});
|
|
597
599
|
} catch (e: any) {
|
|
598
600
|
console.error(`Error getting call id override for ${config.moduleId}.${config.functionId}, falling back to original call id`, { error: e.stack });
|
|
@@ -1096,15 +1096,21 @@ export class Querysub {
|
|
|
1096
1096
|
* BOTH start and end must be provided, otherwise if only one is provided we will ignore it.
|
|
1097
1097
|
*/
|
|
1098
1098
|
endFraction?: number;
|
|
1099
|
+
/** Only listen to keys on this network. If not specified, only keys on the default network are returned. */
|
|
1100
|
+
network?: string;
|
|
1099
1101
|
}): (keyof T)[] {
|
|
1100
1102
|
|
|
1101
|
-
let { startFraction, endFraction } = config || {};
|
|
1103
|
+
let { startFraction, endFraction, network } = config || {};
|
|
1102
1104
|
if ((startFraction === undefined) !== (endFraction === undefined)) {
|
|
1103
1105
|
throw new Error(`startFraction and endFraction must both be provided, or both be undefined. If you want to get all keys, don't provide startFraction and endFraction.`);
|
|
1104
1106
|
}
|
|
1105
1107
|
if (!isNode() && startFraction !== undefined && endFraction !== undefined) {
|
|
1106
1108
|
console.warn(`keys() with a range restriction is not supported clientside. It's too complicated for the proxy to handle it because the hashing depends on the authority server. You can synchronize all the keys client side, but you can't synchronize a restriction of the keys.`);
|
|
1107
1109
|
}
|
|
1110
|
+
if (network && startFraction === undefined) {
|
|
1111
|
+
startFraction = 0;
|
|
1112
|
+
endFraction = 1;
|
|
1113
|
+
}
|
|
1108
1114
|
if (startFraction === undefined || endFraction === undefined) {
|
|
1109
1115
|
return Object.keys(obj);
|
|
1110
1116
|
}
|
|
@@ -1112,10 +1118,15 @@ export class Querysub {
|
|
|
1112
1118
|
if (!path) {
|
|
1113
1119
|
return Object.keys(obj);
|
|
1114
1120
|
}
|
|
1115
|
-
let packedPath = encodeParentFilter({ path, startFraction, endFraction });
|
|
1121
|
+
let packedPath = encodeParentFilter({ path, startFraction, endFraction, network });
|
|
1116
1122
|
return proxyWatcher.getKeys(packedPath);
|
|
1117
1123
|
}
|
|
1118
1124
|
|
|
1125
|
+
/** Creates a key that lives on the given network. Reads and writes using the returned key route to authorities on that network (instead of the default network), and it will only be enumerated by Querysub.keys calls that ask for that network. */
|
|
1126
|
+
public static createNetworkKey(config: { key: string; network: string }): string {
|
|
1127
|
+
return createNetworkKey({ originalKey: config.key, network: config.network });
|
|
1128
|
+
}
|
|
1129
|
+
|
|
1119
1130
|
// TODO: Maybe expose checkPermissions(getValue: () => unknown)?
|
|
1120
1131
|
// - It would be easy, if we every need to explicitly check if we have permissions. Although, it seems
|
|
1121
1132
|
// like just relying on the automatic checking is better?
|
|
@@ -1366,5 +1377,6 @@ import { onAllPredictionsFinished } from "../-0-hooks/hooks";
|
|
|
1366
1377
|
import { LOCAL_DOMAIN } from "../0-path-value-core/PathRouter";
|
|
1367
1378
|
import { authorityLookup } from "../0-path-value-core/AuthorityLookup";
|
|
1368
1379
|
import { encodeParentFilter } from "../0-path-value-core/hackedPackedPathParentFiltering";
|
|
1380
|
+
import { createNetworkKey } from "../0-path-value-core/PathRouterRouteOverride";
|
|
1369
1381
|
import { AliveChecker, registerAliveChecker } from "../2-proxy/garbageCollection";
|
|
1370
1382
|
import { QuerysubController, anyPredictionsPending, flushDelayedFunctions, onCallPredict, waitUntilAllPredictionsFinish } from "./QuerysubController";
|
|
@@ -27,16 +27,16 @@ import { CallerContextBase } from "socket-function/SocketFunctionTypes";
|
|
|
27
27
|
import { isTrustedByNode } from "../-d-trust/NetworkTrust2";
|
|
28
28
|
import { Querysub, id } from "./Querysub";
|
|
29
29
|
import { isDefined } from "../misc";
|
|
30
|
-
import {
|
|
30
|
+
import { isClient, isServer } from "../config2";
|
|
31
31
|
import { PromiseObj } from "../promise";
|
|
32
32
|
import { LoggingClient } from "../0-path-value-core/LoggingClient";
|
|
33
33
|
import * as prediction from "./querysubPrediction";
|
|
34
34
|
setFlag(require, "preact", "allowclient", true);
|
|
35
35
|
|
|
36
36
|
import yargs from "yargs";
|
|
37
|
-
import {
|
|
37
|
+
import { setRoutingOverrideKeyNetwork } from "../0-path-value-core/PathRouterRouteOverride";
|
|
38
38
|
import { isManagementUser, onAllPredictionsFinished } from "../-0-hooks/hooks";
|
|
39
|
-
import { getDomain, isBootstrapOnly } from "../config";
|
|
39
|
+
import { getDomain, getPrimaryNetwork, isBootstrapOnly, DEFAULT_NETWORK } from "../config";
|
|
40
40
|
import { flushPredictionQueueBase, runInPredictionQueue, syncHasPendingPredictionsBase } from "./predictionQueue";
|
|
41
41
|
import { PathRouter } from "../0-path-value-core/PathRouter";
|
|
42
42
|
import { authorityLookup } from "../0-path-value-core/AuthorityLookup";
|
|
@@ -551,10 +551,17 @@ export class QuerysubControllerBase {
|
|
|
551
551
|
let callerCreatorId = IdentityController_getPubKeyShort(caller);
|
|
552
552
|
call.callerIP = IdentityController_getSecureIP(caller);
|
|
553
553
|
|
|
554
|
-
if (call.
|
|
555
|
-
throw new Error(`Caller is not a management user, and so does not have permissions to set
|
|
554
|
+
if (call.network && call.network !== DEFAULT_NETWORK && !await isManagementUser()) {
|
|
555
|
+
throw new Error(`Caller is not a management user, and so does not have permissions to set the network on calls. Call ${debugCallSpec(call)}, network was "${call.network}"`);
|
|
556
|
+
}
|
|
557
|
+
// Clients don't know their network, so we decide it for them, rewriting the callId so the call routes to (and is picked up by) the right network.
|
|
558
|
+
if (!call.network) {
|
|
559
|
+
let network = getPrimaryNetwork();
|
|
560
|
+
if (network !== DEFAULT_NETWORK) {
|
|
561
|
+
call.network = network;
|
|
562
|
+
call.CallId = setRoutingOverrideKeyNetwork(call.CallId, network);
|
|
563
|
+
}
|
|
556
564
|
}
|
|
557
|
-
call.filterable = mergeFilterables([getFncFilter(), call.filterable]);
|
|
558
565
|
|
|
559
566
|
if (Querysub.SIMULATE_LAG) {
|
|
560
567
|
await delay(Querysub.SIMULATE_LAG * 2);
|
package/src/config.ts
CHANGED
|
@@ -5,6 +5,7 @@ import { MaybePromise } from "socket-function/src/types";
|
|
|
5
5
|
import { parseArgsFactory } from "./misc/rawParams";
|
|
6
6
|
import { lazy } from "socket-function/src/caching";
|
|
7
7
|
import fs from "fs";
|
|
8
|
+
import os from "os";
|
|
8
9
|
import { SocketFunction } from "socket-function/SocketFunction";
|
|
9
10
|
|
|
10
11
|
export const serverPort = 11748;
|
|
@@ -32,7 +33,8 @@ let yargObj = parseArgsFactory()
|
|
|
32
33
|
})
|
|
33
34
|
.option("logbackblaze", { type: "boolean", desc: "Log all backblaze activity to disk." })
|
|
34
35
|
.option("slowdown", { type: "number", desc: "Delay all input data values by this amount of time, pretending like we didn't even receive it until this time is up." })
|
|
35
|
-
.option("
|
|
36
|
+
.option("network", { type: "array", desc: `The networks this node is on (pass multiple arguments to be on multiple, ex: --network test --network default). Authorities only satisfy paths on their networks ("default" if unset). Function calls are put on the first network in the list. If no FunctionRunner is on a call's network, the call will fail to run.` })
|
|
37
|
+
.option("networkfile", { type: "string", desc: `The same as --network, except the networks are read from the given file (one per line). Supports "~/" for the home directory. If the file doesn't exist, the process exits with an error.` })
|
|
36
38
|
.argv
|
|
37
39
|
;
|
|
38
40
|
|
|
@@ -42,8 +44,42 @@ if (isNode()) {
|
|
|
42
44
|
}
|
|
43
45
|
}
|
|
44
46
|
|
|
45
|
-
export
|
|
46
|
-
|
|
47
|
+
export const DEFAULT_NETWORK = "default";
|
|
48
|
+
|
|
49
|
+
export function expandHomePath(path: string): string {
|
|
50
|
+
if (path === "~" || path.startsWith("~/") || path.startsWith("~\\")) {
|
|
51
|
+
return os.homedir() + path.slice(1);
|
|
52
|
+
}
|
|
53
|
+
return path;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
let networkFileNetworks = lazy((): string[] => {
|
|
57
|
+
if (!yargObj.networkfile) return [];
|
|
58
|
+
let path = expandHomePath(String(yargObj.networkfile));
|
|
59
|
+
if (!fs.existsSync(path)) {
|
|
60
|
+
console.error(`--networkfile file not found: ${path}`);
|
|
61
|
+
process.exit(1);
|
|
62
|
+
}
|
|
63
|
+
return fs.readFileSync(path, "utf8").split("\n").map(x => x.trim()).filter(x => x);
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
export function getNetworks(): string[] {
|
|
67
|
+
let networks = yargObj.network;
|
|
68
|
+
if (!networks) {
|
|
69
|
+
networks = [];
|
|
70
|
+
} else if (!Array.isArray(networks)) {
|
|
71
|
+
networks = [networks];
|
|
72
|
+
}
|
|
73
|
+
let result = networks.map(x => String(x));
|
|
74
|
+
if (isNode()) {
|
|
75
|
+
result = [...result, ...networkFileNetworks()];
|
|
76
|
+
}
|
|
77
|
+
result = Array.from(new Set(result));
|
|
78
|
+
if (result.length === 0) return [DEFAULT_NETWORK];
|
|
79
|
+
return result;
|
|
80
|
+
}
|
|
81
|
+
export function getPrimaryNetwork(): string {
|
|
82
|
+
return getNetworks()[0];
|
|
47
83
|
}
|
|
48
84
|
|
|
49
85
|
type QuerysubConfig = {
|
package/src/config2.ts
CHANGED
|
@@ -1,12 +1,10 @@
|
|
|
1
1
|
import { deepCloneJSON, isNode } from "socket-function/src/misc";
|
|
2
2
|
import { hasArchivesPermissions } from "./-a-archives/archives";
|
|
3
|
-
import { baseIsClient, getDomain
|
|
3
|
+
import { baseIsClient, getDomain } from "./config";
|
|
4
4
|
import { JSONLACKS } from "socket-function/src/JSONLACKS/JSONLACKS";
|
|
5
5
|
import { rootPathStr, prependToPathStr, getPathDepth } from "./path";
|
|
6
6
|
import fs from "fs";
|
|
7
7
|
import { AuthoritySpec } from "./0-path-value-core/PathRouter";
|
|
8
|
-
import { parseFilterable } from "./misc/filterable";
|
|
9
|
-
import { lazy } from "socket-function/src/caching";
|
|
10
8
|
|
|
11
9
|
export function isClient() {
|
|
12
10
|
return baseIsClient();
|
|
@@ -24,8 +22,4 @@ export function evaluateValidStates() {
|
|
|
24
22
|
return isServer();
|
|
25
23
|
}
|
|
26
24
|
|
|
27
|
-
// NOTE: getOurAuthorities moved to PathRouterHashOverrides, and renamed to getOurAuthoritySpec
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
export const getFncFilter = lazy(() => parseFilterable(getRawFncFilter() || ""));
|
|
25
|
+
// NOTE: getOurAuthorities moved to PathRouterHashOverrides, and renamed to getOurAuthoritySpec
|
|
@@ -7,6 +7,7 @@ import { nextId, sort } from "socket-function/src/misc";
|
|
|
7
7
|
import { formatDateTime, formatTime } from "socket-function/src/formatting/format";
|
|
8
8
|
import { Querysub } from "../../../4-querysub/Querysub";
|
|
9
9
|
import { URLParam } from "../../../library-components/URLParam";
|
|
10
|
+
import { mainResets } from "../../../library-components/urlResetGroups";
|
|
10
11
|
import { ATag } from "../../../library-components/ATag";
|
|
11
12
|
import { Button } from "../../../library-components/Button";
|
|
12
13
|
import { InputLabel } from "../../../library-components/InputLabel";
|
|
@@ -16,8 +17,8 @@ import { TicketsController, watchTickets } from "./tickets";
|
|
|
16
17
|
import { isTicketFinished, Ticket, TicketComment, TicketPatchFile, TicketState, TICKET_STATES } from "./ticketTypes";
|
|
17
18
|
|
|
18
19
|
export const ticketIdURL = new URLParam("ticketid", "");
|
|
19
|
-
// "unfinished" hides tickets in a final state (fixed / not-a-bug).
|
|
20
|
-
export const ticketFilterURL = new URLParam("ticketfilter", "");
|
|
20
|
+
// "unfinished" hides tickets in a final state (fixed / not-a-bug). Resets when the page changes, so the filter doesn't confusingly stick around.
|
|
21
|
+
export const ticketFilterURL = new URLParam("ticketfilter", "", { reset: [mainResets] });
|
|
21
22
|
|
|
22
23
|
const TITLE_MAX_LENGTH = 200;
|
|
23
24
|
const COMMENT_TEXTAREA_MIN_HEIGHT = 250;
|
|
@@ -205,7 +206,7 @@ class TicketList extends qreact.Component {
|
|
|
205
206
|
return <div className={css.vbox(16).pad2(16).fillBoth.minHeight(0)}>
|
|
206
207
|
<div className={css.hbox(16)}>
|
|
207
208
|
<h2>Tickets ({sorted.length}{showUnfinishedOnly && " unfinished" || ""})</h2>
|
|
208
|
-
<ATag values={[ticketFilterURL.getOverride(showUnfinishedOnly
|
|
209
|
+
<ATag values={[ticketFilterURL.getOverride(showUnfinishedOnly ? "" : "unfinished")]}>
|
|
209
210
|
{showUnfinishedOnly && "Show All" || "Show Unfinished Only"}
|
|
210
211
|
</ATag>
|
|
211
212
|
<Button
|
|
@@ -112,7 +112,7 @@ export async function registerManagementPages2(config: {
|
|
|
112
112
|
getModule: () => import("./misc-pages/SnapshotViewer"),
|
|
113
113
|
});
|
|
114
114
|
inputPages.push({
|
|
115
|
-
title: "
|
|
115
|
+
title: "Routing Table",
|
|
116
116
|
componentName: "AuthoritySpecPage",
|
|
117
117
|
controllerName: "AuthoritySpecPageController",
|
|
118
118
|
getModule: () => import("./misc-pages/AuthoritySpecPage"),
|