querysub 0.504.0 → 0.506.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -20,6 +20,8 @@ import { auditLog, isDebugLogEnabled } from "../0-path-value-core/auditLogs";
20
20
  import { debugNodeThread } from "../-c-identity/IdentityController";
21
21
  import { authorityLookup } from "../0-path-value-core/AuthorityLookup";
22
22
  import { decodeParentFilter, encodeParentFilter, registerGetSpecForChildPath } from "../0-path-value-core/hackedPackedPathParentFiltering";
23
+ import { getPathNetwork } from "../0-path-value-core/PathRouterRouteOverride";
24
+ import { DEFAULT_NETWORK } from "../config";
23
25
  import { removeRange, rangesOverlap } from "../rangeMath";
24
26
  import { evaluateValidStates } from "../config2";
25
27
 
@@ -46,8 +48,10 @@ export class RemoteWatcher {
46
48
  start: number;
47
49
  // exclusive
48
50
  end: number;
49
- // encodeParentFilter({ path: cleanPath, startFraction: start, endFraction: end })
51
+ // encodeParentFilter({ path: cleanPath, startFraction: start, endFraction: end, network })
50
52
  finalPath: string;
53
+ // undefined means the default network
54
+ network?: string;
51
55
  // All of the requests paths that request it (pre decodeParentFilter).
52
56
  inputPaths: Set<string>;
53
57
 
@@ -139,8 +143,10 @@ export class RemoteWatcher {
139
143
  let parentPath = getParentPathStr(path);
140
144
  let watchObj = this.remoteWatchParents2.get(parentPath);
141
145
  if (!watchObj) return undefined;
142
- // NOTE: PathRouter.getChildReadNodes PROMISES That all of the nodes will hash in the same way, and if they all hash in the same way, and they have different ranges, then every path will uniquely map to a single value.
146
+ let pathNetwork = getPathNetwork(path);
147
+ // NOTE: PathRouter.getChildReadNodes PROMISES That all of the nodes will hash in the same way, and if they all hash in the same way, and they have different ranges, then every path will uniquely map to a single value.
143
148
  return watchObj.ranges.find(x => {
149
+ if ((x.network || DEFAULT_NETWORK) !== pathNetwork) return false;
144
150
  let route = PathRouter.getRouteFull({ path, spec: x.authoritySpec });
145
151
  return x.start <= route && route < x.end;
146
152
  });
@@ -149,8 +155,10 @@ export class RemoteWatcher {
149
155
  let parentPath = getParentPathStr(path);
150
156
  let watchObj = this.remoteWatchParents2.get(parentPath);
151
157
  if (!watchObj) return undefined;
152
- // NOTE: PathRouter.getChildReadNodes PROMISES That all of the nodes will hash in the same way, and if they all hash in the same way, and they have different ranges, then every path will uniquely map to a single value.
158
+ let pathNetwork = getPathNetwork(path);
159
+ // NOTE: PathRouter.getChildReadNodes PROMISES That all of the nodes will hash in the same way, and if they all hash in the same way, and they have different ranges, then every path will uniquely map to a single value.
153
160
  for (let range of watchObj.ranges) {
161
+ if ((range.network || DEFAULT_NETWORK) !== pathNetwork) continue;
154
162
  let route = PathRouter.getRouteFull({ path, spec: range.authoritySpec });
155
163
  if (range.start <= route && route < range.end) {
156
164
  return route;
@@ -258,17 +266,17 @@ export class RemoteWatcher {
258
266
  if (totalMissingPaths === 0) {
259
267
  let authorities = authorityLookup.getTopologySync();
260
268
  authorityId = PathRouter.getReadyAuthority(path)?.nodeId;
261
- console.warn(`Missing authority for path ${path}`, { authorities });
269
+ console.warn(`Missing authority for path ${path} (network ${getPathNetwork(path)})`, { network: getPathNetwork(path), authorities });
262
270
  }
263
271
  totalMissingPaths++;
264
272
  if (!this.disconnectedPaths.has(path)) {
265
273
  newDisconnectPaths++;
266
274
  this.disconnectedPaths.add(path);
267
- auditLog("NO AUTHORITY FOR PATH", { path });
275
+ auditLog("NO AUTHORITY FOR PATH", { path, network: getPathNetwork(path) });
268
276
  }
269
277
  continue;
270
278
  }
271
- auditLog("FOUND AUTHORITY FOR PATH", { path, authorityId });
279
+ auditLog("FOUND AUTHORITY FOR PATH", { path, network: getPathNetwork(path), authorityId });
272
280
  this.disconnectedPaths.delete(path);
273
281
  foundPaths++;
274
282
 
@@ -289,6 +297,7 @@ export class RemoteWatcher {
289
297
 
290
298
  let decodedParentFilter = decodeParentFilter(path);
291
299
  let cleanPath = decodedParentFilter?.path ?? path;
300
+ let network = decodedParentFilter?.network;
292
301
 
293
302
  let rangeStart = 0;
294
303
  let rangeEnd = 1;
@@ -297,6 +306,8 @@ export class RemoteWatcher {
297
306
  rangeEnd = decodedParentFilter.end;
298
307
  }
299
308
 
309
+ const sameNetwork = (rangeNetwork: string | undefined) => (rangeNetwork || DEFAULT_NETWORK) === (network || DEFAULT_NETWORK);
310
+
300
311
  let watchObj = this.remoteWatchParents2.get(cleanPath);
301
312
 
302
313
  if (watchObj && !config.tryReconnect) {
@@ -306,11 +317,13 @@ export class RemoteWatcher {
306
317
  end: rangeEnd,
307
318
  }];
308
319
  for (let range of watchObj.ranges) {
320
+ if (!sameNetwork(range.network)) continue;
309
321
  removeRange(requiredRanges, range);
310
322
  }
311
323
  if (requiredRanges.length === 0) {
312
324
  this.disconnectedParents.delete(path);
313
325
  for (let range of watchObj.ranges) {
326
+ if (!sameNetwork(range.network)) continue;
314
327
  if (rangesOverlap(range, { start: rangeStart, end: rangeEnd })) {
315
328
  range.inputPaths.add(path);
316
329
  }
@@ -319,7 +332,8 @@ export class RemoteWatcher {
319
332
  }
320
333
  }
321
334
 
322
- let { nodes } = PathRouter.getChildReadNodes(cleanPath, {
335
+ let readNodesPath = network && encodeParentFilter({ path: cleanPath, startFraction: 0, endFraction: 1, network }) || cleanPath;
336
+ let { nodes } = PathRouter.getChildReadNodes(readNodesPath, {
323
337
  // Pass existing connected node ids as preferred
324
338
  preferredNodeIds: watchObj?.ranges.map(x => x.authorityId) || []
325
339
  });
@@ -347,6 +361,7 @@ export class RemoteWatcher {
347
361
  // Compute which parts of [rangeStart, rangeEnd] aren't yet covered by existing watches
348
362
  let missingRanges: { start: number; end: number }[] = [{ start: rangeStart, end: rangeEnd }];
349
363
  for (let range of watchObj.ranges) {
364
+ if (!sameNetwork(range.network)) continue;
350
365
  removeRange(missingRanges, range);
351
366
  }
352
367
 
@@ -359,13 +374,14 @@ export class RemoteWatcher {
359
374
  let clippedStart = Math.max(node.range.start, missing.start);
360
375
  let clippedEnd = Math.min(node.range.end, missing.end);
361
376
 
362
- let finalPath = encodeParentFilter({ path: cleanPath, startFraction: clippedStart, endFraction: clippedEnd });
377
+ let finalPath = encodeParentFilter({ path: cleanPath, startFraction: clippedStart, endFraction: clippedEnd, network });
363
378
  let existingRange = watchObj.ranges.find(x => x.finalPath === finalPath && x.authorityId === node.nodeId);
364
379
  if (!existingRange) {
365
380
  existingRange = {
366
381
  start: clippedStart,
367
382
  end: clippedEnd,
368
383
  finalPath,
384
+ network,
369
385
  inputPaths: new Set(),
370
386
  authorityId: node.nodeId,
371
387
  suppressedWatches: new Set(),
@@ -385,6 +401,7 @@ export class RemoteWatcher {
385
401
 
386
402
  // Register path on pre-existing ranges that already cover parts of [rangeStart, rangeEnd]
387
403
  for (let range of watchObj.ranges) {
404
+ if (!sameNetwork(range.network)) continue;
388
405
  if (rangesOverlap(range, { start: rangeStart, end: rangeEnd })) {
389
406
  range.inputPaths.add(path);
390
407
  }
@@ -396,7 +413,7 @@ export class RemoteWatcher {
396
413
  console.warn(`Some paths have no authority. We will search for an authority periodically until we find an authority. New missing ${newDisconnectPaths} paths and ${newDisconnectParents} parent paths, total missing ${this.disconnectedPaths.size} paths and ${this.disconnectedParents.size} parent paths.`);
397
414
  let first10 = Array.from(this.disconnectedPaths).slice(0, 10);
398
415
  for (let path of first10) {
399
- console.log(`\t${path}`);
416
+ console.log(`\t(network ${getPathNetwork(path)}) ${path}`);
400
417
  }
401
418
  if (this.disconnectedPaths.size > 10) {
402
419
  console.log(`\t... and ${this.disconnectedPaths.size - 10} more paths`);
@@ -416,10 +433,10 @@ export class RemoteWatcher {
416
433
  if (isOwnNodeId(authorityId)) continue;
417
434
 
418
435
  for (let path of paths) {
419
- auditLog("remoteWatcher outer WATCH", { path, remoteNodeId: authorityId });
436
+ auditLog("remoteWatcher outer WATCH", { path, network: getPathNetwork(path), remoteNodeId: authorityId });
420
437
  }
421
438
  for (let path of parentPaths) {
422
- auditLog("remoteWatcher outer PARENT WATCH", { path, remoteNodeId: authorityId });
439
+ auditLog("remoteWatcher outer PARENT WATCH", { path, network: decodeParentFilter(path)?.network || DEFAULT_NETWORK, remoteNodeId: authorityId });
423
440
  }
424
441
 
425
442
  if (STOP_KEYS_DOUBLE_SENDS) {
@@ -428,7 +445,7 @@ export class RemoteWatcher {
428
445
  let range = this.getRemoteWatchParentRange(path);
429
446
  if (range) {
430
447
  range.suppressedWatches.add(path);
431
- auditLog("remoteWatcher outer WATCH SUPPRESSED", { path, remoteNodeId: authorityId });
448
+ auditLog("remoteWatcher outer WATCH SUPPRESSED", { path, network: getPathNetwork(path), remoteNodeId: authorityId });
432
449
  return false;
433
450
  }
434
451
  return true;
@@ -480,10 +497,10 @@ export class RemoteWatcher {
480
497
  }
481
498
  for (let [authorityId, { paths, parentPaths }] of byAuthority) {
482
499
  for (let path of paths) {
483
- auditLog("remoteWatcher inner WATCH", { path, remoteNodeId: authorityId });
500
+ auditLog("remoteWatcher inner WATCH", { path, network: getPathNetwork(path), remoteNodeId: authorityId });
484
501
  }
485
502
  for (let path of parentPaths) {
486
- auditLog("remoteWatcher inner WATCH PARENT", { path, remoteNodeId: authorityId });
503
+ auditLog("remoteWatcher inner WATCH PARENT", { path, network: decodeParentFilter(path)?.network || DEFAULT_NETWORK, remoteNodeId: authorityId });
487
504
  }
488
505
  // NOTE: We log watches here because we want to log batched watches
489
506
  ActionsHistory.OnNewWatches({ newPathsWatched: paths, newParentsWatched: parentPaths, debugName: batched[0].debugName, authorityId });
@@ -494,10 +511,10 @@ export class RemoteWatcher {
494
511
  };
495
512
  if (isDebugLogEnabled()) {
496
513
  for (let path of paths) {
497
- auditLog("Asking to watch path", { path, authorityId, targetNodeThreadId: debugNodeThread(authorityId) });
514
+ auditLog("Asking to watch path", { path, network: getPathNetwork(path), authorityId, targetNodeThreadId: debugNodeThread(authorityId) });
498
515
  }
499
516
  for (let path of parentPaths) {
500
- auditLog("Asking to watch parent path", { path, authorityId, targetNodeThreadId: debugNodeThread(authorityId) });
517
+ auditLog("Asking to watch parent path", { path, network: decodeParentFilter(path)?.network || DEFAULT_NETWORK, authorityId, targetNodeThreadId: debugNodeThread(authorityId) });
501
518
  }
502
519
  }
503
520
  logErrors(RemoteWatcher.REMOTE_WATCH_FUNCTION(config, authorityId));
@@ -587,10 +604,10 @@ export class RemoteWatcher {
587
604
  }
588
605
  for (let [authorityIdBase, { paths, parentPaths }] of unwatchesPerAuthority.entries()) {
589
606
  for (let path of paths) {
590
- auditLog("remoteWatcher inner UNWATCH", { path, remoteNodeId: authorityIdBase });
607
+ auditLog("remoteWatcher inner UNWATCH", { path, network: getPathNetwork(path), remoteNodeId: authorityIdBase });
591
608
  }
592
609
  for (let path of parentPaths) {
593
- auditLog("remoteWatcher inner UNWATCH PARENT", { path, remoteNodeId: authorityIdBase });
610
+ auditLog("remoteWatcher inner UNWATCH PARENT", { path, network: decodeParentFilter(path)?.network || DEFAULT_NETWORK, remoteNodeId: authorityIdBase });
594
611
  }
595
612
  if (!isOwnNodeId(authorityIdBase)) {
596
613
  logErrors(RemoteWatcher.REMOTE_UNWATCH_FUNCTION({ paths, parentPaths }, authorityIdBase));
@@ -14,13 +14,12 @@ import { blue, green, red } from "socket-function/src/formatting/logColors";
14
14
  import { getPathStr2 } from "../path";
15
15
  import { isNode, sort } from "socket-function/src/misc";
16
16
  import { decodeCborx, encodeCborx } from "../misc/cloneHelpers";
17
- import { parseFilterable } from "../misc/filterable";
18
17
  import { interceptCalls } from "../-0-hooks/hooks";
19
18
  import { createRoutingOverrideKey } from "../0-path-value-core/PathRouterRouteOverride";
20
19
  import { PathRouter } from "../0-path-value-core/PathRouter";
21
20
  import { getPathFromProxy } from "../2-proxy/pathValueProxy";
22
- import { getDomain, isPublic } from "../config";
23
- import { getFncFilter, isServer } from "../config2";
21
+ import { getDomain, getPrimaryNetwork, isPublic, DEFAULT_NETWORK } from "../config";
22
+ import { isServer } from "../config2";
24
23
  import { getTimeUnique } from "socket-function/src/bits";
25
24
 
26
25
  // NOTE: We could deploy single functions, but... we will almost always be updating all functions at
@@ -161,16 +160,27 @@ export function writeFunctionCall(config: {
161
160
  // which gives them about a minute to guess it...
162
161
  let callId = now + "_" + secureRandom();
163
162
 
163
+ // Clients don't know their network. The QuerysubController decides the network for their calls (and rewrites the callId to match).
164
+ let network: string | undefined;
165
+ if (isNode()) {
166
+ network = getPrimaryNetwork();
167
+ if (network === DEFAULT_NETWORK) {
168
+ network = undefined;
169
+ }
170
+ }
171
+
164
172
  // We HAVE to override the routing id, otherwise PathFunctionRunner secondary sharding won't work
165
173
  callId = getCallIdOverride({
166
174
  moduleId,
167
175
  functionId,
168
176
  callId,
169
177
  args,
178
+ network,
170
179
  }) || createRoutingOverrideKey({
171
180
  originalKey: callId,
172
181
  routeKey: callId,
173
182
  remappedPrefix: getFunctionRunnerPrefix(moduleId),
183
+ network,
174
184
  });
175
185
 
176
186
  let argsEncoded = encodeArgs(args);
@@ -185,19 +195,8 @@ export function writeFunctionCall(config: {
185
195
  // Will be updated by Querysub to be correct (unless we are directly writing, then...
186
196
  // this is fine).
187
197
  callerIP: "127.0.0.1",
198
+ network,
188
199
  };
189
- // NOTE: I don't know when this would ever really work. I guess to isolate function calls, but if we're running locally, isn't that easy?
190
- // if (!isNode()) {
191
- // // Get the "setfncfilter" querystring parameter
192
- // let url = new URL(window.location.href);
193
- // let setfncfilter = url.searchParams.get("setfncfilter");
194
- // if (setfncfilter) {
195
- // callSpec.filterable = parseFilterable(setfncfilter);
196
- // }
197
- // }
198
- if (isNode()) {
199
- callSpec.filterable = getFncFilter();
200
- }
201
200
 
202
201
  if (curInterceptor) {
203
202
  curInterceptor.onCall(callSpec, metadata);
@@ -16,10 +16,9 @@ import { parseArgs } from "./PathFunctionHelpers";
16
16
  import { FunctionMetadata, PERMISSIONS_FUNCTION_ID, addRoutingPrefixForDeploy, getAllDevelopmentModulesIds, getDevelopmentModule, getExportPath, getModuleRelativePath, getSchemaObject } from "./syncSchema";
17
17
  import { formatTime } from "socket-function/src/formatting/format";
18
18
  import { getControllerNodeIdList, set_debug_getFunctionRunnerShards } from "../-g-core-values/NodeCapabilities";
19
- import { FilterSelector, Filterable, doesMatch } from "../misc/filterable";
20
19
  import { SocketFunction } from "socket-function/SocketFunction";
21
20
  import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
22
- import { getDomain, isPublic } from "../config";
21
+ import { getDomain, isPublic, DEFAULT_NETWORK } from "../config";
23
22
  import { getGitRefSync, getGitURLSync } from "../4-deploy/git";
24
23
  import type { DeployProgress } from "../4-deploy/deployFunctions";
25
24
  import { getRoutingOverride, getRoutingOverridePart } from "../0-path-value-core/PathRouterRouteOverride";
@@ -105,10 +104,11 @@ export interface CallSpec {
105
104
  callerIP: string;
106
105
  runAtTime: Time;
107
106
 
108
- // Not just used for debugging, also used to add special proxy-related warnings.
107
+ // Not just used for debugging, also used to add special proxy-related warnings.
109
108
  fromProxy?: string;
110
109
 
111
- filterable?: Filterable;
110
+ // The network the call is on ("default" if unset). Only FunctionRunners on this network will run the call.
111
+ network?: string;
112
112
  }
113
113
  export function debugCallSpec(spec: CallSpec): string {
114
114
  return `${spec.DomainName}/${spec.ModuleId}/${spec.FunctionId}`;
@@ -227,7 +227,8 @@ export class PathFunctionRunner {
227
227
  shardRange: { startFraction: number, endFraction: number };
228
228
  secondaryShardRange?: { startFraction: number, endFraction: number };
229
229
  PermissionsChecker: PermissionsCheckType | undefined;
230
- filterSelector?: FilterSelector;
230
+ // The networks we listen to calls on. Unset is equivalent to ["default"].
231
+ networks?: string[];
231
232
  }) {
232
233
  SocketFunction.expose(FunctionPreloadController);
233
234
  SocketFunction.expose(FunctionCaptureController);
@@ -264,11 +265,14 @@ export class PathFunctionRunner {
264
265
 
265
266
  let outstandingCalls = 0;
266
267
 
268
+ let networks = this.config.networks && this.config.networks.length > 0 && this.config.networks || [DEFAULT_NETWORK];
269
+
267
270
  let watchModuleCalls = cache((moduleId: string) => {
268
- // Object.keys(functionSchema()[domainName].PathFunctionRunner[moduleId].Calls);
269
- // Object.keys(functionSchema()[domainName].PathFunctionRunner[moduleId].Results);
270
- Querysub.keys(functionSchema()[domainName].PathFunctionRunner[moduleId].Calls, fullFraction);
271
- Querysub.keys(functionSchema()[domainName].PathFunctionRunner[moduleId].Results, fullFraction);
271
+ for (let network of networks) {
272
+ let networkConfig = network === DEFAULT_NETWORK && fullFraction || { ...fullFraction, network };
273
+ Querysub.keys(functionSchema()[domainName].PathFunctionRunner[moduleId].Calls, networkConfig);
274
+ Querysub.keys(functionSchema()[domainName].PathFunctionRunner[moduleId].Results, networkConfig);
275
+ }
272
276
  });
273
277
 
274
278
  let runningCalls = new Set<string>();
@@ -394,8 +398,8 @@ export class PathFunctionRunner {
394
398
  }
395
399
 
396
400
  if (PathFunctionRunner.DEBUG_CALLS) {
397
- // NOTE: If this start, it means that the filter didn't match, which either means it was a dev function and so not relevant for us, or we are dev and it's a production function, so not relevant for us.
398
- // - The filter is set by fncfilter when running serverside, or, clientside, by fncfilter set on the command line of the querysub server.
401
+ // NOTE: If this starts but never runs, it means the call's network didn't match ours, which either means it was a dev call and so not relevant for us, or we are dev and it's a production call, so not relevant for us.
402
+ // - The network is set by --network when running serverside, or, clientside, by --network set on the command line of the querysub server.
399
403
  console.log(`QUEUING ${getDebugName(callData, functionSpec, true)}`);
400
404
  let resultsPath = getProxyPath(() => moduleData.Results[callId]);
401
405
  let history = authorityStorage.getValuePlusHistory(resultsPath);
@@ -551,7 +555,9 @@ export class PathFunctionRunner {
551
555
  skipPermissions = PermissionsChecker.skipPermissionsChecks.bind(PermissionsChecker);
552
556
  }
553
557
 
554
- if (!doesMatch(callSpec.filterable, this.config.filterSelector)) {
558
+ let callNetwork = callSpec.network || DEFAULT_NETWORK;
559
+ let ourNetworks = this.config.networks && this.config.networks.length > 0 && this.config.networks || [DEFAULT_NETWORK];
560
+ if (!ourNetworks.includes(callNetwork)) {
555
561
  return;
556
562
  }
557
563
 
@@ -6,7 +6,6 @@ import yargs from "yargs";
6
6
  let yargObj = yargs(process.argv)
7
7
  .option("fncshard", { type: "string", default: "0-1", desc: "Shard range as start-end (e.g., '0-0.5' or '0.25-0.75')" })
8
8
  .option("fncsecshard", { type: "string", default: "", desc: "Secondary shard range as start-end (e.g., '0-0.5'). Values in this range (not matched in fncshard) are delayed." })
9
- .option("filter", { type: "string", default: "", desc: `Filter to only include handle specific function calls (fncshard is still applied). For example, "a&b|c", using regular boolean rules.` })
10
9
  .argv
11
10
  ;
12
11
 
@@ -16,9 +15,8 @@ import { SocketFunction } from "socket-function/SocketFunction";
16
15
  import { getThreadKeyCert } from "sliftutils/misc/https/certs";
17
16
  import { ClientWatcher } from "../1-path-client/pathValueClientWatcher";
18
17
  import { timeInMinute } from "socket-function/src/misc";
19
- import { getDomain, isLocal, isPublic } from "../config";
18
+ import { getDomain, getNetworks, isLocal, isPublic, DEFAULT_NETWORK } from "../config";
20
19
  import { green, magenta } from "socket-function/src/formatting/logColors";
21
- import { parseFilterSelector } from "../misc/filterable";
22
20
  import path from "path";
23
21
  import { IndexedLogs } from "../diagnostics/logs/IndexedLogs/IndexedLogs";
24
22
 
@@ -61,7 +59,7 @@ async function main() {
61
59
  console.log(green(`Sharding from ${shardStart} to ${shardEnd}`));
62
60
  }
63
61
 
64
- let filterSelector = parseFilterSelector(yargObj.filter);
62
+ let networks = getNetworks();
65
63
 
66
64
  new PathFunctionRunner({
67
65
  domainName: getDomain(),
@@ -69,7 +67,7 @@ async function main() {
69
67
  secondaryShardRange,
70
68
  // TODO: Maybe abstract this out even more, so anything can plug in permissions checks?
71
69
  PermissionsChecker: PermissionsCheck,
72
- filterSelector,
70
+ networks,
73
71
  });
74
72
 
75
73
  if (!isPublic()) {
@@ -78,8 +76,8 @@ async function main() {
78
76
  await import(deployPath);
79
77
  }
80
78
 
81
- if (yargObj.filter) {
82
- console.log(magenta(`Only running functions that match the filter: ${yargObj.filter}. Either use --fncfilter dev in your http server to can calls here, or use ?setfncfilter=${yargObj.filter} in the browser to trigger functions that match this filter.`));
79
+ if (networks.length !== 1 || networks[0] !== DEFAULT_NETWORK) {
80
+ console.log(magenta(`Only running functions on the network(s): ${networks.join(", ")}. Use --network ${networks[0]} in your http server to route calls here.`));
83
81
  }
84
82
  }
85
83
  logErrors(main());
@@ -579,11 +579,12 @@ export function getCallIdOverride(config: {
579
579
  functionId: string;
580
580
  callId: string;
581
581
  args: unknown[];
582
+ network?: string;
582
583
  }): string {
583
584
  try {
584
585
  if (querysub) {
585
586
  if (!querysub.Querysub.isInSyncedCall()) {
586
- // NOTE: This is wrong and it will result in loading user being used as a key sometimes. However, it should be fine, as if the call id override is wrong, it shouldn't break anything.
587
+ // NOTE: This is wrong and it will result in loading user being used as a key sometimes. However, it should be fine, as if the call id override is wrong, it shouldn't break anything.
587
588
  return querysub.Querysub.localRead(() => getCallIdOverride(config));
588
589
  }
589
590
  }
@@ -593,6 +594,7 @@ export function getCallIdOverride(config: {
593
594
  remappedPrefix: def.prefix,
594
595
  originalKey: config.callId,
595
596
  routeKey: def.getKey(...config.args),
597
+ network: config.network,
596
598
  });
597
599
  } catch (e: any) {
598
600
  console.error(`Error getting call id override for ${config.moduleId}.${config.functionId}, falling back to original call id`, { error: e.stack });
@@ -1096,15 +1096,21 @@ export class Querysub {
1096
1096
  * BOTH start and end must be provided, otherwise if only one is provided we will ignore it.
1097
1097
  */
1098
1098
  endFraction?: number;
1099
+ /** Only listen to keys on this network. If not specified, only keys on the default network are returned. */
1100
+ network?: string;
1099
1101
  }): (keyof T)[] {
1100
1102
 
1101
- let { startFraction, endFraction } = config || {};
1103
+ let { startFraction, endFraction, network } = config || {};
1102
1104
  if ((startFraction === undefined) !== (endFraction === undefined)) {
1103
1105
  throw new Error(`startFraction and endFraction must both be provided, or both be undefined. If you want to get all keys, don't provide startFraction and endFraction.`);
1104
1106
  }
1105
1107
  if (!isNode() && startFraction !== undefined && endFraction !== undefined) {
1106
1108
  console.warn(`keys() with a range restriction is not supported clientside. It's too complicated for the proxy to handle it because the hashing depends on the authority server. You can synchronize all the keys client side, but you can't synchronize a restriction of the keys.`);
1107
1109
  }
1110
+ if (network && startFraction === undefined) {
1111
+ startFraction = 0;
1112
+ endFraction = 1;
1113
+ }
1108
1114
  if (startFraction === undefined || endFraction === undefined) {
1109
1115
  return Object.keys(obj);
1110
1116
  }
@@ -1112,10 +1118,15 @@ export class Querysub {
1112
1118
  if (!path) {
1113
1119
  return Object.keys(obj);
1114
1120
  }
1115
- let packedPath = encodeParentFilter({ path, startFraction, endFraction });
1121
+ let packedPath = encodeParentFilter({ path, startFraction, endFraction, network });
1116
1122
  return proxyWatcher.getKeys(packedPath);
1117
1123
  }
1118
1124
 
1125
+ /** Creates a key that lives on the given network. Reads and writes using the returned key route to authorities on that network (instead of the default network), and it will only be enumerated by Querysub.keys calls that ask for that network. */
1126
+ public static createNetworkKey(config: { key: string; network: string }): string {
1127
+ return createNetworkKey({ originalKey: config.key, network: config.network });
1128
+ }
1129
+
1119
1130
  // TODO: Maybe expose checkPermissions(getValue: () => unknown)?
1120
1131
  // - It would be easy, if we every need to explicitly check if we have permissions. Although, it seems
1121
1132
  // like just relying on the automatic checking is better?
@@ -1366,5 +1377,6 @@ import { onAllPredictionsFinished } from "../-0-hooks/hooks";
1366
1377
  import { LOCAL_DOMAIN } from "../0-path-value-core/PathRouter";
1367
1378
  import { authorityLookup } from "../0-path-value-core/AuthorityLookup";
1368
1379
  import { encodeParentFilter } from "../0-path-value-core/hackedPackedPathParentFiltering";
1380
+ import { createNetworkKey } from "../0-path-value-core/PathRouterRouteOverride";
1369
1381
  import { AliveChecker, registerAliveChecker } from "../2-proxy/garbageCollection";
1370
1382
  import { QuerysubController, anyPredictionsPending, flushDelayedFunctions, onCallPredict, waitUntilAllPredictionsFinish } from "./QuerysubController";
@@ -27,16 +27,16 @@ import { CallerContextBase } from "socket-function/SocketFunctionTypes";
27
27
  import { isTrustedByNode } from "../-d-trust/NetworkTrust2";
28
28
  import { Querysub, id } from "./Querysub";
29
29
  import { isDefined } from "../misc";
30
- import { getFncFilter, isClient, isServer } from "../config2";
30
+ import { isClient, isServer } from "../config2";
31
31
  import { PromiseObj } from "../promise";
32
32
  import { LoggingClient } from "../0-path-value-core/LoggingClient";
33
33
  import * as prediction from "./querysubPrediction";
34
34
  setFlag(require, "preact", "allowclient", true);
35
35
 
36
36
  import yargs from "yargs";
37
- import { mergeFilterables, parseFilterable, serializeFilterable } from "../misc/filterable";
37
+ import { setRoutingOverrideKeyNetwork } from "../0-path-value-core/PathRouterRouteOverride";
38
38
  import { isManagementUser, onAllPredictionsFinished } from "../-0-hooks/hooks";
39
- import { getDomain, isBootstrapOnly } from "../config";
39
+ import { getDomain, getPrimaryNetwork, isBootstrapOnly, DEFAULT_NETWORK } from "../config";
40
40
  import { flushPredictionQueueBase, runInPredictionQueue, syncHasPendingPredictionsBase } from "./predictionQueue";
41
41
  import { PathRouter } from "../0-path-value-core/PathRouter";
42
42
  import { authorityLookup } from "../0-path-value-core/AuthorityLookup";
@@ -551,10 +551,17 @@ export class QuerysubControllerBase {
551
551
  let callerCreatorId = IdentityController_getPubKeyShort(caller);
552
552
  call.callerIP = IdentityController_getSecureIP(caller);
553
553
 
554
- if (call.filterable && !await isManagementUser()) {
555
- throw new Error(`Caller is not a management user, and so does not have permissions to set filterable on calls. Call ${debugCallSpec(call)}, filter was "${serializeFilterable(call.filterable)}"`);
554
+ if (call.network && call.network !== DEFAULT_NETWORK && !await isManagementUser()) {
555
+ throw new Error(`Caller is not a management user, and so does not have permissions to set the network on calls. Call ${debugCallSpec(call)}, network was "${call.network}"`);
556
+ }
557
+ // Clients don't know their network, so we decide it for them, rewriting the callId so the call routes to (and is picked up by) the right network.
558
+ if (!call.network) {
559
+ let network = getPrimaryNetwork();
560
+ if (network !== DEFAULT_NETWORK) {
561
+ call.network = network;
562
+ call.CallId = setRoutingOverrideKeyNetwork(call.CallId, network);
563
+ }
556
564
  }
557
- call.filterable = mergeFilterables([getFncFilter(), call.filterable]);
558
565
 
559
566
  if (Querysub.SIMULATE_LAG) {
560
567
  await delay(Querysub.SIMULATE_LAG * 2);
package/src/config.ts CHANGED
@@ -5,6 +5,7 @@ import { MaybePromise } from "socket-function/src/types";
5
5
  import { parseArgsFactory } from "./misc/rawParams";
6
6
  import { lazy } from "socket-function/src/caching";
7
7
  import fs from "fs";
8
+ import os from "os";
8
9
  import { SocketFunction } from "socket-function/SocketFunction";
9
10
 
10
11
  export const serverPort = 11748;
@@ -32,7 +33,8 @@ let yargObj = parseArgsFactory()
32
33
  })
33
34
  .option("logbackblaze", { type: "boolean", desc: "Log all backblaze activity to disk." })
34
35
  .option("slowdown", { type: "number", desc: "Delay all input data values by this amount of time, pretending like we didn't even receive it until this time is up." })
35
- .option("fncfilter", { type: "string", default: "", desc: `Sets the filterable state for function calls, causing them to target specific FunctionRunners. If no FunctionRunner matches, all functions will fail to run. For example: "devtestserver" will match a FunctionRunner that uses the "devtestserver" filter. Merges with the existing filterable state if a client sets it explicitly.` })
36
+ .option("network", { type: "array", desc: `The networks this node is on (pass multiple arguments to be on multiple, ex: --network test --network default). Authorities only satisfy paths on their networks ("default" if unset). Function calls are put on the first network in the list. If no FunctionRunner is on a call's network, the call will fail to run.` })
37
+ .option("networkfile", { type: "string", desc: `The same as --network, except the networks are read from the given file (one per line). Supports "~/" for the home directory. If the file doesn't exist, the process exits with an error.` })
36
38
  .argv
37
39
  ;
38
40
 
@@ -42,8 +44,42 @@ if (isNode()) {
42
44
  }
43
45
  }
44
46
 
45
- export function getRawFncFilter() {
46
- return yargObj.fncfilter;
47
+ export const DEFAULT_NETWORK = "default";
48
+
49
+ export function expandHomePath(path: string): string {
50
+ if (path === "~" || path.startsWith("~/") || path.startsWith("~\\")) {
51
+ return os.homedir() + path.slice(1);
52
+ }
53
+ return path;
54
+ }
55
+
56
+ let networkFileNetworks = lazy((): string[] => {
57
+ if (!yargObj.networkfile) return [];
58
+ let path = expandHomePath(String(yargObj.networkfile));
59
+ if (!fs.existsSync(path)) {
60
+ console.error(`--networkfile file not found: ${path}`);
61
+ process.exit(1);
62
+ }
63
+ return fs.readFileSync(path, "utf8").split("\n").map(x => x.trim()).filter(x => x);
64
+ });
65
+
66
+ export function getNetworks(): string[] {
67
+ let networks = yargObj.network;
68
+ if (!networks) {
69
+ networks = [];
70
+ } else if (!Array.isArray(networks)) {
71
+ networks = [networks];
72
+ }
73
+ let result = networks.map(x => String(x));
74
+ if (isNode()) {
75
+ result = [...result, ...networkFileNetworks()];
76
+ }
77
+ result = Array.from(new Set(result));
78
+ if (result.length === 0) return [DEFAULT_NETWORK];
79
+ return result;
80
+ }
81
+ export function getPrimaryNetwork(): string {
82
+ return getNetworks()[0];
47
83
  }
48
84
 
49
85
  type QuerysubConfig = {
package/src/config2.ts CHANGED
@@ -1,12 +1,10 @@
1
1
  import { deepCloneJSON, isNode } from "socket-function/src/misc";
2
2
  import { hasArchivesPermissions } from "./-a-archives/archives";
3
- import { baseIsClient, getDomain, getRawFncFilter } from "./config";
3
+ import { baseIsClient, getDomain } from "./config";
4
4
  import { JSONLACKS } from "socket-function/src/JSONLACKS/JSONLACKS";
5
5
  import { rootPathStr, prependToPathStr, getPathDepth } from "./path";
6
6
  import fs from "fs";
7
7
  import { AuthoritySpec } from "./0-path-value-core/PathRouter";
8
- import { parseFilterable } from "./misc/filterable";
9
- import { lazy } from "socket-function/src/caching";
10
8
 
11
9
  export function isClient() {
12
10
  return baseIsClient();
@@ -24,8 +22,4 @@ export function evaluateValidStates() {
24
22
  return isServer();
25
23
  }
26
24
 
27
- // NOTE: getOurAuthorities moved to PathRouterHashOverrides, and renamed to getOurAuthoritySpec
28
-
29
-
30
-
31
- export const getFncFilter = lazy(() => parseFilterable(getRawFncFilter() || ""));
25
+ // NOTE: getOurAuthorities moved to PathRouterHashOverrides, and renamed to getOurAuthoritySpec
@@ -7,6 +7,7 @@ import { nextId, sort } from "socket-function/src/misc";
7
7
  import { formatDateTime, formatTime } from "socket-function/src/formatting/format";
8
8
  import { Querysub } from "../../../4-querysub/Querysub";
9
9
  import { URLParam } from "../../../library-components/URLParam";
10
+ import { mainResets } from "../../../library-components/urlResetGroups";
10
11
  import { ATag } from "../../../library-components/ATag";
11
12
  import { Button } from "../../../library-components/Button";
12
13
  import { InputLabel } from "../../../library-components/InputLabel";
@@ -16,8 +17,8 @@ import { TicketsController, watchTickets } from "./tickets";
16
17
  import { isTicketFinished, Ticket, TicketComment, TicketPatchFile, TicketState, TICKET_STATES } from "./ticketTypes";
17
18
 
18
19
  export const ticketIdURL = new URLParam("ticketid", "");
19
- // "unfinished" hides tickets in a final state (fixed / not-a-bug).
20
- export const ticketFilterURL = new URLParam("ticketfilter", "");
20
+ // "unfinished" hides tickets in a final state (fixed / not-a-bug). Resets when the page changes, so the filter doesn't confusingly stick around.
21
+ export const ticketFilterURL = new URLParam("ticketfilter", "", { reset: [mainResets] });
21
22
 
22
23
  const TITLE_MAX_LENGTH = 200;
23
24
  const COMMENT_TEXTAREA_MIN_HEIGHT = 250;
@@ -205,7 +206,7 @@ class TicketList extends qreact.Component {
205
206
  return <div className={css.vbox(16).pad2(16).fillBoth.minHeight(0)}>
206
207
  <div className={css.hbox(16)}>
207
208
  <h2>Tickets ({sorted.length}{showUnfinishedOnly && " unfinished" || ""})</h2>
208
- <ATag values={[ticketFilterURL.getOverride(showUnfinishedOnly && "" || "unfinished")]}>
209
+ <ATag values={[ticketFilterURL.getOverride(showUnfinishedOnly ? "" : "unfinished")]}>
209
210
  {showUnfinishedOnly && "Show All" || "Show Unfinished Only"}
210
211
  </ATag>
211
212
  <Button
@@ -112,7 +112,7 @@ export async function registerManagementPages2(config: {
112
112
  getModule: () => import("./misc-pages/SnapshotViewer"),
113
113
  });
114
114
  inputPages.push({
115
- title: "Authority Specs",
115
+ title: "Routing Table",
116
116
  componentName: "AuthoritySpecPage",
117
117
  controllerName: "AuthoritySpecPageController",
118
118
  getModule: () => import("./misc-pages/AuthoritySpecPage"),