querysub 0.407.0 → 0.409.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/bin/audit-disk-values.js +7 -0
  2. package/package.json +4 -3
  3. package/src/-a-archives/archiveCache.ts +12 -9
  4. package/src/-a-auth/certs.ts +1 -1
  5. package/src/-c-identity/IdentityController.ts +9 -1
  6. package/src/-f-node-discovery/NodeDiscovery.ts +63 -8
  7. package/src/0-path-value-core/AuthorityLookup.ts +8 -3
  8. package/src/0-path-value-core/PathRouter.ts +109 -68
  9. package/src/0-path-value-core/PathRouterServerAuthoritySpec.tsx +4 -2
  10. package/src/0-path-value-core/PathValueCommitter.ts +3 -1
  11. package/src/0-path-value-core/PathValueController.ts +75 -4
  12. package/src/0-path-value-core/PathWatcher.ts +39 -0
  13. package/src/0-path-value-core/ShardPrefixes.ts +2 -0
  14. package/src/0-path-value-core/ValidStateComputer.ts +20 -8
  15. package/src/0-path-value-core/hackedPackedPathParentFiltering.ts +11 -29
  16. package/src/0-path-value-core/pathValueArchives.ts +16 -5
  17. package/src/0-path-value-core/pathValueCore.ts +43 -3
  18. package/src/1-path-client/RemoteWatcher.ts +46 -25
  19. package/src/4-querysub/Querysub.ts +17 -5
  20. package/src/4-querysub/QuerysubController.ts +21 -10
  21. package/src/4-querysub/predictionQueue.tsx +3 -0
  22. package/src/4-querysub/querysubPrediction.ts +27 -20
  23. package/src/5-diagnostics/nodeMetadata.ts +17 -0
  24. package/src/deployManager/components/MachineDetailPage.tsx +1 -1
  25. package/src/diagnostics/NodeConnectionsPage.tsx +167 -0
  26. package/src/diagnostics/NodeViewer.tsx +11 -15
  27. package/src/diagnostics/PathDistributionInfo.tsx +102 -0
  28. package/src/diagnostics/auditDiskValues.ts +221 -0
  29. package/src/diagnostics/auditDiskValuesEntry.ts +43 -0
  30. package/src/diagnostics/logs/IndexedLogs/LogViewer3.tsx +5 -1
  31. package/src/diagnostics/logs/TimeRangeSelector.tsx +3 -3
  32. package/src/diagnostics/logs/lifeCycleAnalysis/LifeCycleRenderer.tsx +2 -0
  33. package/src/diagnostics/managementPages.tsx +10 -1
  34. package/src/diagnostics/misc-pages/ArchiveViewer.tsx +3 -2
  35. package/src/diagnostics/pathAuditer.ts +21 -0
  36. package/src/storage/diskCache2.ts +6 -0
  37. package/tempnotes.txt +9 -44
  38. package/test.ts +13 -301
  39. package/src/diagnostics/benchmark.ts +0 -139
  40. package/src/diagnostics/runSaturationTest.ts +0 -416
  41. package/src/diagnostics/satSchema.ts +0 -64
  42. package/src/test/mongoSatTest.tsx +0 -55
  43. package/src/test/satTest.ts +0 -193
  44. package/src/test/test.tsx +0 -552
@@ -177,6 +177,7 @@ class PathValueCommitter {
177
177
  auditLog("CREATE VALUE", {
178
178
  path: pathValue.path,
179
179
  timeId: pathValue.time.time,
180
+ timeIdFull: pathValue.time,
180
181
  source: pathValue.source,
181
182
  transparent: pathValue.isTransparent,
182
183
  // value: valueStr,
@@ -242,7 +243,7 @@ class PathValueCommitter {
242
243
  void forwardPromise.then(x => {
243
244
  if (x === "refused") {
244
245
  values = values.map(value => {
245
- console.info(`Rejecting past value due to initial sync: ${debugPathValuePath(value)}`, {
246
+ console.info(`Rejecting value that was refused: ${debugPathValuePath(value)}`, {
246
247
  path: value.path,
247
248
  timeId: value.time.time,
248
249
  });
@@ -312,6 +313,7 @@ class PathValueCommitter {
312
313
  let candidates = PathRouter.getAllAuthorities(path);
313
314
  require("debugbreak")(2);
314
315
  debugger;
316
+ remoteWatcher.getExistingWatchRemoteNodeId(path);
315
317
  console.warn(`Ignoring value from wrong authority. Should have been ${debugNodeId(watchingAuthorityId)}, but was received from ${debugNodeId(batch.sourceNodeId)}.`, {
316
318
  path,
317
319
  type,
@@ -7,7 +7,6 @@ import { ActionsHistory } from "../diagnostics/ActionsHistory";
7
7
  import { isNode, nextId, timeInSecond, timeoutToUndefined } from "socket-function/src/misc";
8
8
  import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
9
9
  import { pathValueCommitter } from "./PathValueCommitter";
10
- import { Benchmark } from "../diagnostics/benchmark";
11
10
  import { auditLog, isDebugLogEnabled } from "./auditLogs";
12
11
  import { debugNodeId, debugNodeThread } from "../-c-identity/IdentityController";
13
12
  import { isDiskAudit } from "../config";
@@ -17,8 +16,18 @@ import { getNodeIdIP } from "socket-function/src/nodeCache";
17
16
  import { authorityLookup } from "./AuthorityLookup";
18
17
  import { timeoutToError } from "../errors";
19
18
  import { AuthoritySpec, PathRouter } from "./PathRouter";
19
+ import { LZ4 } from "../storage/LZ4";
20
+ import { Time } from "./pathValueCore";
21
+ import { encodeCborx, decodeCborx } from "../misc/cloneHelpers";
22
+ import { debugGetAllCallFactories } from "socket-function/src/nodeCache";
20
23
  export { pathValueCommitter };
21
24
 
25
+ // ONLY returns non-canGCValues that are valid
26
+ export type AuditSnapshotEntry = {
27
+ path: string;
28
+ time: Time;
29
+ };
30
+
22
31
  // The sync test page has some functions which related to PathValueController significantly, and should always be exposed if PathValueController is exposed.
23
32
  if (isNode()) {
24
33
  setImmediate(() => {
@@ -43,6 +52,7 @@ export class PathValueControllerBase {
43
52
  auditLog("SEND CREATED VALUE", {
44
53
  path: value.path,
45
54
  timeId: value.time.time,
55
+ timeIdVersion: value.time.version,
46
56
  targetNodeThreadId: debugNodeThread(nodeId),
47
57
  transparent: value.isTransparent,
48
58
  source: value.source,
@@ -66,6 +76,7 @@ export class PathValueControllerBase {
66
76
  auditLog("SEND VALUE", {
67
77
  path: pathValue.path,
68
78
  timeId: pathValue.time.time,
79
+ timeIdVersion: pathValue.time.version,
69
80
  watcher: nodeId,
70
81
  isValid: pathValue.valid,
71
82
  nodeId: debugNodeId(nodeId),
@@ -119,7 +130,6 @@ export class PathValueControllerBase {
119
130
  let values: PathValue[] = [];
120
131
  if (valueBuffers) {
121
132
  values = await pathValueSerializer.deserialize(valueBuffers);
122
- Benchmark.onReceivedValues(values);
123
133
  ActionsHistory.OnRead(values);
124
134
  }
125
135
 
@@ -152,6 +162,7 @@ export class PathValueControllerBase {
152
162
  path: value.path,
153
163
  // NOTE: Yes, this might conflict as we're throwing away a lot of information, however in practice no, it's not going to conflict when we're debugging, and this + path will be globally unique for every value we investigate.
154
164
  timeId: value.time.time,
165
+ timeIdVersion: value.time.version,
155
166
  sourceNodeId,
156
167
  sourceNodeThreadId: decodeNodeId(sourceNodeId)?.threadId,
157
168
  });
@@ -252,6 +263,65 @@ export class PathValueControllerBase {
252
263
  });
253
264
  return buffers;
254
265
  }
266
+
267
+ public static async getAuditSnapshot(config: {
268
+ nodeId: string;
269
+ spec: AuthoritySpec;
270
+ }): Promise<AuditSnapshotEntry[]> {
271
+ let compressed = await PathValueController.nodes[config.nodeId].getAuditSnapshot(config);
272
+ let decompressed = LZ4.decompress(compressed);
273
+ return decodeCborx(decompressed);
274
+ }
275
+
276
+ public async getAuditSnapshot(config: {
277
+ spec: AuthoritySpec;
278
+ }): Promise<Buffer> {
279
+ let { spec } = config;
280
+ let { compareTime } = await import("./pathValueCore");
281
+
282
+ let allValues = authorityStorage.getAllValues({ spec, startTime: 0, endTime: Number.MAX_SAFE_INTEGER });
283
+
284
+ let pathToLatest = new Map<string, Time>();
285
+ for (let value of allValues) {
286
+ if (!value.valid) continue;
287
+ if (value.canGCValue) continue;
288
+
289
+ let existing = pathToLatest.get(value.path);
290
+ if (!existing || compareTime(value.time, existing) > 0) {
291
+ pathToLatest.set(value.path, value.time);
292
+ }
293
+ }
294
+
295
+ let entries: AuditSnapshotEntry[] = Array.from(pathToLatest.entries()).map(([path, time]) => ({ path, time }));
296
+
297
+ let encoded = encodeCborx(entries);
298
+ return LZ4.compress(encoded);
299
+ }
300
+
301
+
302
+ public static async getValuesByPathAndTime(config: {
303
+ nodeId: string;
304
+ entries: AuditSnapshotEntry[];
305
+ }): Promise<PathValue[]> {
306
+ let buffers = await PathValueController.nodes[config.nodeId].getValuesByPathAndTime(config.entries);
307
+ return await pathValueSerializer.deserialize(buffers);
308
+ }
309
+
310
+ public async getValuesByPathAndTime(entries: AuditSnapshotEntry[]): Promise<Buffer[]> {
311
+ let values: PathValue[] = [];
312
+ for (let entry of entries) {
313
+ let value = authorityStorage.getValueExactIgnoreInvalid(entry.path, entry.time);
314
+ if (!value) continue;
315
+ if (value.isTransparent) continue;
316
+ if (value.canGCValue) continue;
317
+ values.push(value);
318
+ }
319
+ let buffers = await pathValueSerializer.serialize(values, {
320
+ noLocks: true,
321
+ compress: getCompressNetwork(),
322
+ });
323
+ return buffers;
324
+ }
255
325
  }
256
326
 
257
327
  export const PathValueController = SocketFunction.register(
@@ -261,6 +331,8 @@ export const PathValueController = SocketFunction.register(
261
331
  sendData: {},
262
332
 
263
333
  getInitialValues: {},
334
+ getAuditSnapshot: {},
335
+ getValuesByPathAndTime: {},
264
336
 
265
337
  watchLatest: {},
266
338
  unwatchLatest: {},
@@ -271,5 +343,4 @@ export const PathValueController = SocketFunction.register(
271
343
  {
272
344
  noFunctionMeasure: !isNode(),
273
345
  }
274
- );
275
-
346
+ );
@@ -12,13 +12,43 @@ import { PathRouter } from "./PathRouter";
12
12
  import { auditLog } from "./auditLogs";
13
13
  import { WatchConfig, authorityStorage, PathValue, NodeId, compareTime, isCoreQuiet, MAX_CHANGE_AGE, createMissingEpochValue } from "./pathValueCore";
14
14
  import { matchesParentRangeFilter } from "./hackedPackedPathParentFiltering";
15
+ import { isClient } from "../config2";
16
+ import { delay } from "socket-function/src/batching";
15
17
 
18
+ setImmediate(() => import("../4-querysub/Querysub"));
16
19
 
17
20
  // These are used so we can have our internal trigger lookup without watching them on remote values.
18
21
  export function getInternalValueWatchPrefix() {
19
22
  return "INTERNAL_VALUE_WATCH_PREFIX_";
20
23
  }
21
24
 
25
+ let pathWatcherState: (() => { watcherSequence: number }) | undefined;
26
+
27
+ setImmediate(async () => {
28
+ const { createLocalSchema } = await import("../4-querysub/schemaHelpers");
29
+ const { t } = await import("../2-proxy/schema2");
30
+ pathWatcherState = createLocalSchema("pathWatcherState", {
31
+ watcherSequence: t.number
32
+ });
33
+ });
34
+
35
+ let triggering = false;
36
+ function incrementWatcherSequence() {
37
+ if (!isClient()) return;
38
+ if (!pathWatcherState) return;
39
+ if (triggering) return;
40
+ triggering = true;
41
+ void delay(10).finally(async () => {
42
+ let { Querysub } = await import("../4-querysub/Querysub");
43
+ triggering = false;
44
+ Querysub.commit(() => {
45
+ if (!pathWatcherState) return;
46
+ let state = pathWatcherState();
47
+ state.watcherSequence = (state.watcherSequence || 0) + 1;
48
+ });
49
+ });
50
+ }
51
+
22
52
  // WATCH CASES
23
53
  // 1) getOwnNodeId() is used to trigger pathValueClientWatcher (no one else should be using it)
24
54
  // 2) Use other nodes to immediately proxy to the other nodes
@@ -133,6 +163,7 @@ class PathWatcher {
133
163
  }
134
164
 
135
165
  if (newPathsWatched.size > 0 || newParentsWatched.size > 0) {
166
+ incrementWatcherSequence();
136
167
  if (isOwnNodeId(config.nodeId)) {
137
168
  for (let path of newPathsWatched) {
138
169
  auditLog("new local WATCH VALUE", { path });
@@ -269,6 +300,7 @@ class PathWatcher {
269
300
  }
270
301
 
271
302
  if (fullyUnwatched.paths.length > 0 || fullyUnwatched.parentPaths.length > 0) {
303
+ incrementWatcherSequence();
272
304
  console.info(`Unwatched PathValue watches`, {
273
305
  unwatchedPaths: fullyUnwatched.paths.length,
274
306
  unwatchedParents: fullyUnwatched.parentPaths.length,
@@ -505,6 +537,13 @@ class PathWatcher {
505
537
  return Array.from(this.parentWatchers.keys());
506
538
  }
507
539
 
540
+ public getAllWatchedPaths(): string[] {
541
+ if (pathWatcherState) {
542
+ pathWatcherState().watcherSequence;
543
+ }
544
+ return Array.from(this.watchers.keys());
545
+ }
546
+
508
547
  public debug_harvestSyncTimes(): { start: number; end: number; path: string; }[] {
509
548
  let times = this.syncHistoryForHarvest.getAllUnordered();
510
549
  this.syncHistoryForHarvest.reset();
@@ -9,6 +9,7 @@
9
9
  After a PathServerValue starts, it can't change its prefixes, as if the routing changes it makes it too difficult for clients to reason about who they should be synchronizing with.
10
10
  */
11
11
 
12
+ import { isNode } from "typesafecss";
12
13
  import { nestArchives } from "../-a-archives/archives";
13
14
  import { getArchivesBackblaze } from "../-a-archives/archivesBackBlaze";
14
15
  import { archiveJSONT } from "../-a-archives/archivesJSONT";
@@ -23,6 +24,7 @@ let prefixes = archiveJSONT<PrefixObj>(() => nestArchives("shard-prefixes/", get
23
24
  const key = "all";
24
25
 
25
26
  export async function getShardPrefixes(): Promise<string[]> {
27
+ if (!isNode()) return [];
26
28
  return (await prefixes.get(key))?.prefixes ?? [];
27
29
  }
28
30
 
@@ -5,7 +5,7 @@ import { isDiskAudit } from "../config";
5
5
  import { getPathFromStr } from "../path";
6
6
  import { auditLog } from "./auditLogs";
7
7
  import { PathRouter } from "./PathRouter";
8
- import { PathValue, authorityStorage, compareTime, debugPathValuePath, ReadLock, byLockGroup, isCoreQuiet, debugRejections, debugTime, debugPathValue } from "./pathValueCore";
8
+ import { PathValue, authorityStorage, compareTime, debugPathValuePath, ReadLock, byLockGroup, isCoreQuiet, debugRejections, debugTime, debugPathValue, MAX_CHANGE_AGE } from "./pathValueCore";
9
9
  import { pathWatcher } from "./PathWatcher";
10
10
  import { lockWatcher2 } from "./LockWatcher2";
11
11
  import { onPathInteracted } from "../diagnostics/pathAuditerCallback";
@@ -82,7 +82,9 @@ class ValidStateComputer {
82
82
  console.info(`Rejecting past value due to initial sync: ${debugPathValuePath(value)}`, {
83
83
  path: value.path,
84
84
  timeId: value.time.time,
85
+ timeIdFull: value.time,
85
86
  newTimeId: newValue?.time.time,
87
+ newTimeIdFull: newValue?.time,
86
88
  });
87
89
  ourValues.push({ ...value, valid: false, });
88
90
  }
@@ -158,10 +160,11 @@ class ValidStateComputer {
158
160
  }
159
161
  return !!value.valid;
160
162
  }
161
- private isLockContentionFree(lock: ReadLock): PathValue | undefined {
163
+ private isLockContentionFree(lock: ReadLock): PathValue[] | undefined {
162
164
  // sorted by -time (newest are first)
163
165
  let history = authorityStorage.getValuePlusHistory(lock.path);
164
166
 
167
+ let conflictingValues: PathValue[] | undefined;
165
168
  let index = binarySearchIndex(history.length, i => compareTime(lock.endTime, history[i].time));
166
169
  if (index < 0) index = ~index;
167
170
  else index++;
@@ -172,10 +175,11 @@ class ValidStateComputer {
172
175
  if (value.isTransparent && lock.readIsTransparent) continue;
173
176
  // If value.time is within our range, there's contention!
174
177
  if (compareTime(lock.startTime, value.time) < 0 && compareTime(value.time, lock.endTime) < 0) {
175
- return value;
178
+ if (!conflictingValues) conflictingValues = [];
179
+ conflictingValues.push(value);
176
180
  }
177
181
  }
178
- return undefined;
182
+ return conflictingValues;
179
183
  }
180
184
 
181
185
  @measureFnc
@@ -193,6 +197,10 @@ class ValidStateComputer {
193
197
 
194
198
  let valid = true;
195
199
  for (let lock of locks) {
200
+ let age = now - lock.endTime.time;
201
+ // If it's old enough, then it's always valid, we can't change the valid save something after its max change age, even if it's invalid.
202
+ if (age > MAX_CHANGE_AGE) continue;
203
+
196
204
  if (!this.isLockValid(lock)) {
197
205
  onPathInteracted(lock.path, 2);
198
206
  valid = false;
@@ -259,7 +267,7 @@ class ValidStateComputer {
259
267
  }
260
268
  }
261
269
  }
262
- private logLockContention(valueGroup: PathValue[], lock: ReadLock, conflictingValue: PathValue, now: number) {
270
+ private logLockContention(valueGroup: PathValue[], lock: ReadLock, conflictingValues: PathValue[], now: number) {
263
271
 
264
272
  // This special version indicates clientside prediction (which SHOULD be rejected).
265
273
  if (lock.endTime.version !== Number.MAX_SAFE_INTEGER) {
@@ -278,7 +286,7 @@ class ValidStateComputer {
278
286
  redMessage += `\n (original read from: ${debugTime(lock.startTime)}`;
279
287
  redMessage += `\n (at time: ${debugTime(lock.endTime)}`;
280
288
  redMessage += `\n (current time: ${Date.now()})`;
281
- redMessage += `\n (conflict write at: ${debugTime(conflictingValue.time)}`;
289
+ redMessage += `\n (conflict write at: ${debugTime(conflictingValues[0].time)}`;
282
290
  console.error(redMessage);
283
291
  }
284
292
  if (isDiskAudit()) {
@@ -287,11 +295,15 @@ class ValidStateComputer {
287
295
  lock,
288
296
  path: pathValue.path,
289
297
  timeId: pathValue.time.time,
298
+ timeIdVersion: pathValue.time.version,
299
+ timeIdFull: pathValue.time,
290
300
  lockedPath: lock.path,
291
301
  transparent: lock.readIsTransparent,
292
302
  missingPath: lock.path,
293
- conflictTimeId0: conflictingValue.time.time,
294
- conflictingWrites: conflictingValue,
303
+ conflictTimeId0: conflictingValues[0].time.time,
304
+ conflictTimeIdVersion0: conflictingValues[0].time.version,
305
+ conflictCount: conflictingValues.length,
306
+ conflictingWrites: conflictingValues,
295
307
  hadTimeId: lock.startTime.time,
296
308
  });
297
309
  }
@@ -1,22 +1,15 @@
1
1
  import { measureWrap } from "socket-function/src/profiling/measure";
2
2
  import { getPathDepth, getPathIndexAssert, hack_getPackedPathSuffix, hack_setPackedPathSuffix, hack_stripPackedPath } from "../path";
3
- import { PathRouter } from "./PathRouter";
3
+ import { AuthoritySpec, PathRouter } from "./PathRouter";
4
4
  import { cache } from "socket-function/src/caching";
5
5
  import { authorityLookup } from "./AuthorityLookup";
6
6
 
7
- // NOTE: This code has been moved into PathRouter
8
- // /** Returns a number between 0 (inclusive) and 1 (exclusive)
9
- // * - See matchesParentRangeFilter, matchesParentRangeFilterPart, and filterChildPaths.
10
- // */
11
- // export function __getRoutingHash(key: string): number {
12
- // // Using fastHash is about twice as fast as sha256 (although that might be due to the buffer allocation?)
13
- // let hash = fastHash(key);
14
- // return hash % (1000 * 1000 * 1000) / (1000 * 1000 * 1000);
15
- // // let hash = sha256(key);
16
- // // return getBufferFraction(Buffer.from(hash, "hex"));
17
- // }
18
7
 
19
- // NOTE: Assumes fullPath is already a child of parentPath, and only checks for hash
8
+ let getSpecForChildPath = (path: string) => authorityLookup.getOurSpec();
9
+ export function registerGetSpecForChildPath(fnc: (path: string) => AuthoritySpec) {
10
+ getSpecForChildPath = fnc;
11
+ }
12
+
20
13
  export function matchesParentRangeFilter(config: {
21
14
  parentPath: string;
22
15
  fullPath: string;
@@ -26,33 +19,22 @@ export function matchesParentRangeFilter(config: {
26
19
  if (config.parentPath === config.packedPath) return true;
27
20
  let filter = decodeParentFilter(config.packedPath);
28
21
  if (!filter) return true;
29
- let route = PathRouter.getRouteChildKey(config.fullPath);
22
+ if (filter.start <= 0 && filter.end >= 1) return true;
23
+ let route = PathRouter.getRouteFull({ path: config.fullPath, spec: getSpecForChildPath(config.fullPath) });
30
24
  return filter.start <= route && route < filter.end;
31
25
  }
32
- // export function matchesParentRangeFilterPart(config: {
33
- // part: string;
34
- // start: number;
35
- // end: number;
36
- // }) {
37
- // if (config.start === 0 && config.end === 1) return true;
38
- // let hash = __getRoutingHash(config.part);
39
- // return config.start <= hash && hash < config.end;
40
- // }
41
26
 
42
- // IMPORTANT! Child hashing is always based on the last key, never based on the full path. We need to make it always consistent, and we want it to work for parallel values. It cannot change depending on the routing, as this then becomes extremely complicated for proxies and for determining if we are actually supposed to be receiving a value from that node, and for the node to know what values it should send, etc.
43
27
  export const filterChildPathsBase = measureWrap(
44
28
  function filterChildPathsBase(parentPath: string, packedSuffix: string, paths: Set<string>): Set<string> {
45
29
  let [startFractionStr, endFractionStr] = packedSuffix.split("|");
46
30
  let startFraction = Number(startFractionStr);
47
31
  let endFraction = Number(endFractionStr);
48
32
 
49
- let depth = getPathDepth(parentPath);
50
-
51
33
  let filtered = new Set<string>();
52
34
  for (let path of paths) {
53
- let key = getPathIndexAssert(path, depth);
54
- let hash = PathRouter.getSingleKeyRoute(key);
55
- if (startFraction <= hash && hash < endFraction) {
35
+ // TODO: We can make this significantly more efficient as once we know if it's a prefix or not, we know the underlying function call every time. However, that starts to get complicated, and I don't think this is going to be a bottleneck.
36
+ let route = PathRouter.getRouteFull({ path, spec: getSpecForChildPath(path) });
37
+ if (startFraction <= route && route < endFraction) {
56
38
  filtered.add(path);
57
39
  }
58
40
  }
@@ -223,6 +223,7 @@ export class PathValueArchives {
223
223
  let totalSize = 0;
224
224
 
225
225
  let dataPaths: string[] = [];
226
+ let allPaths: string[] = [];
226
227
  let readCache = new Map<string, Buffer>();
227
228
 
228
229
  let downloadStartTime = Date.now();
@@ -233,8 +234,12 @@ export class PathValueArchives {
233
234
  // the source file, but by now it has moved, so we don't see either file, even though at all times
234
235
  // both files existed).
235
236
  let time = Date.now();
236
- dataPaths = await this.getValuePaths(authority);
237
- console.log(green(`${dataPaths.length} data paths in ${formatTime(Date.now() - time)}`));
237
+ let valueObjs = await this.getValuePaths(authority);
238
+ dataPaths = valueObjs.pickedPaths;
239
+ allPaths = valueObjs.allPaths;
240
+ let allValueCount = allPaths.reduce((acc, path) => acc + this.decodeDataPath(path).valueCount, 0);
241
+ let pickedValueCount = dataPaths.reduce((acc, path) => acc + this.decodeDataPath(path).valueCount, 0);
242
+ console.log(green(`${dataPaths.length}/${allPaths.length} (values ${formatNumber(pickedValueCount)} / ${formatNumber(allValueCount)}) data paths in ${formatTime(Date.now() - time)}`));
238
243
  // NOTE: If the notMatched count is high enough, it is possible NodePathAuthorities.ts:authoritiesMightOverlap is
239
244
  // too loose, and should be removing more cases.
240
245
 
@@ -437,7 +442,10 @@ export class PathValueArchives {
437
442
  }
438
443
 
439
444
  @measureFnc
440
- public async getValuePaths(authority: AuthoritySpec): Promise<string[]> {
445
+ public async getValuePaths(authority: AuthoritySpec): Promise<{
446
+ pickedPaths: string[];
447
+ allPaths: string[];
448
+ }> {
441
449
  // let paths: string[] = [];
442
450
  // let pickedDirectories = await this.getAuthorityDirs(authority);
443
451
 
@@ -450,8 +458,11 @@ export class PathValueArchives {
450
458
 
451
459
  let locker = await this.getArchiveLocker();
452
460
  let allFiles = (await locker.getAllValidFiles()).map(x => x.file);
453
- let doesMatch = cache((dir: string) => PathRouter.overlapsPathIdentifier(authority, dir));
454
- return allFiles.filter(x => doesMatch(x.split("/").slice(0, -1).join("/")));
461
+ let pickedPaths = allFiles.filter(x => PathRouter.overlapsPathIdentifier(authority, x));
462
+ return {
463
+ pickedPaths,
464
+ allPaths: allFiles,
465
+ };
455
466
  }
456
467
  @measureFnc
457
468
  public async getValuePathSizes(paths: string[]): Promise<number[]> {
@@ -24,6 +24,8 @@ import { fastHash } from "../misc/hash";
24
24
  import { authorityLookup } from "./AuthorityLookup";
25
25
  import { onPathInteracted } from "../diagnostics/pathAuditerCallback";
26
26
  import { decodeParentFilter, filterChildPathsBase } from "./hackedPackedPathParentFiltering";
27
+ import { isDiskAudit } from "../config";
28
+ import { removeRange } from "../rangeMath";
27
29
 
28
30
  setImmediate(async () => {
29
31
  // Import everything will dynamically import, so the client side can tell that it's required.
@@ -543,6 +545,9 @@ class AuthorityPathValueStorage {
543
545
  if (this.DEBUG_UNWATCH) {
544
546
  console.log(blue(`Unsyncing path at ${Date.now()}`), path);
545
547
  }
548
+ if (isDiskAudit()) {
549
+ auditLog("DESTROY PATH", { path });
550
+ }
546
551
 
547
552
  this.isSyncedCache.delete(path);
548
553
  this.removePathFromStorage(path, "unwatched");
@@ -604,11 +609,45 @@ class AuthorityPathValueStorage {
604
609
  return false;
605
610
  }
606
611
  public isParentSynced(path: string) {
612
+ let range = decodeParentFilter(path);
607
613
  path = hack_stripPackedPath(path);
608
- // We have to check a test child path to see if we are a self authority
609
- if (PathRouter.isSelfAuthority(appendToPathStr(path, ""))) return true;
614
+ if (PathRouter.isLocalPath(path)) return true;
615
+
616
+ let synced = this.parentsSynced.get(path);
617
+ if (synced === true) return true;
618
+ // See if the ranges received so far cover the requested range. If we only request a partial range, then this will always be the case. We'll never fully synchronize the path.
619
+ if (synced && range) {
620
+ let missingRanges: { start: number; end: number }[] = [{ start: range.start, end: range.end }];
621
+ for (let range of synced) {
622
+ removeRange(missingRanges, range);
623
+ }
624
+ if (missingRanges.length === 0) return true;
625
+ }
610
626
 
611
- return this.parentsSynced.get(path) === true;
627
+ // Self authority check
628
+ {
629
+ let nodes = PathRouter.getChildReadNodes(path, { onlyOwnNodes: true });
630
+ if (nodes.nodes.length > 0) {
631
+ if (nodes.nodes.every(x => isOwnNodeId(x.nodeId))) {
632
+ // If it's not a partial 30 and we're self authority, we're always going to be self authority, so we can just cache it in the lookup.
633
+ this.parentsSynced.set(path, true);
634
+ return true;
635
+ }
636
+ // Might be a partial authority
637
+ if (range) {
638
+ let selfNodes = nodes.nodes.filter(x => isOwnNodeId(x.nodeId));
639
+ let missingRanges: { start: number; end: number }[] = [{ start: range.start, end: range.end }];
640
+ for (let range of selfNodes) {
641
+ removeRange(missingRanges, range.range);
642
+ }
643
+ if (missingRanges.length === 0) {
644
+ return true;
645
+ }
646
+ }
647
+ }
648
+ }
649
+
650
+ return false;
612
651
  }
613
652
 
614
653
  /** Obviously just for debugging. Doesn't trigger any watchers, just erases it as if we never had it. There shouldn't be really any downstream caches within the same process at least, and so this should result in the next read having the value missing. If we in the future add downstream caches, we're going to have to invalidate them as well.
@@ -1052,6 +1091,7 @@ class AuthorityPathValueStorage {
1052
1091
  let { spec, startTime, endTime } = config;
1053
1092
  let result: PathValue[] = [];
1054
1093
  for (let [path, values] of this.values) {
1094
+ if (PathRouter.isLocalPath(path)) continue;
1055
1095
  if (PathRouter.matchesAuthoritySpec(spec, path)) {
1056
1096
  for (let value of values) {
1057
1097
  let time = value.time.time;