querysub 0.407.0 → 0.409.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/audit-disk-values.js +7 -0
- package/package.json +4 -3
- package/src/-a-archives/archiveCache.ts +12 -9
- package/src/-a-auth/certs.ts +1 -1
- package/src/-c-identity/IdentityController.ts +9 -1
- package/src/-f-node-discovery/NodeDiscovery.ts +63 -8
- package/src/0-path-value-core/AuthorityLookup.ts +8 -3
- package/src/0-path-value-core/PathRouter.ts +109 -68
- package/src/0-path-value-core/PathRouterServerAuthoritySpec.tsx +4 -2
- package/src/0-path-value-core/PathValueCommitter.ts +3 -1
- package/src/0-path-value-core/PathValueController.ts +75 -4
- package/src/0-path-value-core/PathWatcher.ts +39 -0
- package/src/0-path-value-core/ShardPrefixes.ts +2 -0
- package/src/0-path-value-core/ValidStateComputer.ts +20 -8
- package/src/0-path-value-core/hackedPackedPathParentFiltering.ts +11 -29
- package/src/0-path-value-core/pathValueArchives.ts +16 -5
- package/src/0-path-value-core/pathValueCore.ts +43 -3
- package/src/1-path-client/RemoteWatcher.ts +46 -25
- package/src/4-querysub/Querysub.ts +17 -5
- package/src/4-querysub/QuerysubController.ts +21 -10
- package/src/4-querysub/predictionQueue.tsx +3 -0
- package/src/4-querysub/querysubPrediction.ts +27 -20
- package/src/5-diagnostics/nodeMetadata.ts +17 -0
- package/src/deployManager/components/MachineDetailPage.tsx +1 -1
- package/src/diagnostics/NodeConnectionsPage.tsx +167 -0
- package/src/diagnostics/NodeViewer.tsx +11 -15
- package/src/diagnostics/PathDistributionInfo.tsx +102 -0
- package/src/diagnostics/auditDiskValues.ts +221 -0
- package/src/diagnostics/auditDiskValuesEntry.ts +43 -0
- package/src/diagnostics/logs/IndexedLogs/LogViewer3.tsx +5 -1
- package/src/diagnostics/logs/TimeRangeSelector.tsx +3 -3
- package/src/diagnostics/logs/lifeCycleAnalysis/LifeCycleRenderer.tsx +2 -0
- package/src/diagnostics/managementPages.tsx +10 -1
- package/src/diagnostics/misc-pages/ArchiveViewer.tsx +3 -2
- package/src/diagnostics/pathAuditer.ts +21 -0
- package/src/storage/diskCache2.ts +6 -0
- package/tempnotes.txt +9 -44
- package/test.ts +13 -301
- package/src/diagnostics/benchmark.ts +0 -139
- package/src/diagnostics/runSaturationTest.ts +0 -416
- package/src/diagnostics/satSchema.ts +0 -64
- package/src/test/mongoSatTest.tsx +0 -55
- package/src/test/satTest.ts +0 -193
- package/src/test/test.tsx +0 -552
|
@@ -177,6 +177,7 @@ class PathValueCommitter {
|
|
|
177
177
|
auditLog("CREATE VALUE", {
|
|
178
178
|
path: pathValue.path,
|
|
179
179
|
timeId: pathValue.time.time,
|
|
180
|
+
timeIdFull: pathValue.time,
|
|
180
181
|
source: pathValue.source,
|
|
181
182
|
transparent: pathValue.isTransparent,
|
|
182
183
|
// value: valueStr,
|
|
@@ -242,7 +243,7 @@ class PathValueCommitter {
|
|
|
242
243
|
void forwardPromise.then(x => {
|
|
243
244
|
if (x === "refused") {
|
|
244
245
|
values = values.map(value => {
|
|
245
|
-
console.info(`Rejecting
|
|
246
|
+
console.info(`Rejecting value that was refused: ${debugPathValuePath(value)}`, {
|
|
246
247
|
path: value.path,
|
|
247
248
|
timeId: value.time.time,
|
|
248
249
|
});
|
|
@@ -312,6 +313,7 @@ class PathValueCommitter {
|
|
|
312
313
|
let candidates = PathRouter.getAllAuthorities(path);
|
|
313
314
|
require("debugbreak")(2);
|
|
314
315
|
debugger;
|
|
316
|
+
remoteWatcher.getExistingWatchRemoteNodeId(path);
|
|
315
317
|
console.warn(`Ignoring value from wrong authority. Should have been ${debugNodeId(watchingAuthorityId)}, but was received from ${debugNodeId(batch.sourceNodeId)}.`, {
|
|
316
318
|
path,
|
|
317
319
|
type,
|
|
@@ -7,7 +7,6 @@ import { ActionsHistory } from "../diagnostics/ActionsHistory";
|
|
|
7
7
|
import { isNode, nextId, timeInSecond, timeoutToUndefined } from "socket-function/src/misc";
|
|
8
8
|
import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
|
|
9
9
|
import { pathValueCommitter } from "./PathValueCommitter";
|
|
10
|
-
import { Benchmark } from "../diagnostics/benchmark";
|
|
11
10
|
import { auditLog, isDebugLogEnabled } from "./auditLogs";
|
|
12
11
|
import { debugNodeId, debugNodeThread } from "../-c-identity/IdentityController";
|
|
13
12
|
import { isDiskAudit } from "../config";
|
|
@@ -17,8 +16,18 @@ import { getNodeIdIP } from "socket-function/src/nodeCache";
|
|
|
17
16
|
import { authorityLookup } from "./AuthorityLookup";
|
|
18
17
|
import { timeoutToError } from "../errors";
|
|
19
18
|
import { AuthoritySpec, PathRouter } from "./PathRouter";
|
|
19
|
+
import { LZ4 } from "../storage/LZ4";
|
|
20
|
+
import { Time } from "./pathValueCore";
|
|
21
|
+
import { encodeCborx, decodeCborx } from "../misc/cloneHelpers";
|
|
22
|
+
import { debugGetAllCallFactories } from "socket-function/src/nodeCache";
|
|
20
23
|
export { pathValueCommitter };
|
|
21
24
|
|
|
25
|
+
// ONLY returns non-canGCValues that are valid
|
|
26
|
+
export type AuditSnapshotEntry = {
|
|
27
|
+
path: string;
|
|
28
|
+
time: Time;
|
|
29
|
+
};
|
|
30
|
+
|
|
22
31
|
// The sync test page has some functions which related to PathValueController significantly, and should always be exposed if PathValueController is exposed.
|
|
23
32
|
if (isNode()) {
|
|
24
33
|
setImmediate(() => {
|
|
@@ -43,6 +52,7 @@ export class PathValueControllerBase {
|
|
|
43
52
|
auditLog("SEND CREATED VALUE", {
|
|
44
53
|
path: value.path,
|
|
45
54
|
timeId: value.time.time,
|
|
55
|
+
timeIdVersion: value.time.version,
|
|
46
56
|
targetNodeThreadId: debugNodeThread(nodeId),
|
|
47
57
|
transparent: value.isTransparent,
|
|
48
58
|
source: value.source,
|
|
@@ -66,6 +76,7 @@ export class PathValueControllerBase {
|
|
|
66
76
|
auditLog("SEND VALUE", {
|
|
67
77
|
path: pathValue.path,
|
|
68
78
|
timeId: pathValue.time.time,
|
|
79
|
+
timeIdVersion: pathValue.time.version,
|
|
69
80
|
watcher: nodeId,
|
|
70
81
|
isValid: pathValue.valid,
|
|
71
82
|
nodeId: debugNodeId(nodeId),
|
|
@@ -119,7 +130,6 @@ export class PathValueControllerBase {
|
|
|
119
130
|
let values: PathValue[] = [];
|
|
120
131
|
if (valueBuffers) {
|
|
121
132
|
values = await pathValueSerializer.deserialize(valueBuffers);
|
|
122
|
-
Benchmark.onReceivedValues(values);
|
|
123
133
|
ActionsHistory.OnRead(values);
|
|
124
134
|
}
|
|
125
135
|
|
|
@@ -152,6 +162,7 @@ export class PathValueControllerBase {
|
|
|
152
162
|
path: value.path,
|
|
153
163
|
// NOTE: Yes, this might conflict as we're throwing away a lot of information, however in practice no, it's not going to conflict when we're debugging, and this + path will be globally unique for every value we investigate.
|
|
154
164
|
timeId: value.time.time,
|
|
165
|
+
timeIdVersion: value.time.version,
|
|
155
166
|
sourceNodeId,
|
|
156
167
|
sourceNodeThreadId: decodeNodeId(sourceNodeId)?.threadId,
|
|
157
168
|
});
|
|
@@ -252,6 +263,65 @@ export class PathValueControllerBase {
|
|
|
252
263
|
});
|
|
253
264
|
return buffers;
|
|
254
265
|
}
|
|
266
|
+
|
|
267
|
+
public static async getAuditSnapshot(config: {
|
|
268
|
+
nodeId: string;
|
|
269
|
+
spec: AuthoritySpec;
|
|
270
|
+
}): Promise<AuditSnapshotEntry[]> {
|
|
271
|
+
let compressed = await PathValueController.nodes[config.nodeId].getAuditSnapshot(config);
|
|
272
|
+
let decompressed = LZ4.decompress(compressed);
|
|
273
|
+
return decodeCborx(decompressed);
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
public async getAuditSnapshot(config: {
|
|
277
|
+
spec: AuthoritySpec;
|
|
278
|
+
}): Promise<Buffer> {
|
|
279
|
+
let { spec } = config;
|
|
280
|
+
let { compareTime } = await import("./pathValueCore");
|
|
281
|
+
|
|
282
|
+
let allValues = authorityStorage.getAllValues({ spec, startTime: 0, endTime: Number.MAX_SAFE_INTEGER });
|
|
283
|
+
|
|
284
|
+
let pathToLatest = new Map<string, Time>();
|
|
285
|
+
for (let value of allValues) {
|
|
286
|
+
if (!value.valid) continue;
|
|
287
|
+
if (value.canGCValue) continue;
|
|
288
|
+
|
|
289
|
+
let existing = pathToLatest.get(value.path);
|
|
290
|
+
if (!existing || compareTime(value.time, existing) > 0) {
|
|
291
|
+
pathToLatest.set(value.path, value.time);
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
let entries: AuditSnapshotEntry[] = Array.from(pathToLatest.entries()).map(([path, time]) => ({ path, time }));
|
|
296
|
+
|
|
297
|
+
let encoded = encodeCborx(entries);
|
|
298
|
+
return LZ4.compress(encoded);
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
public static async getValuesByPathAndTime(config: {
|
|
303
|
+
nodeId: string;
|
|
304
|
+
entries: AuditSnapshotEntry[];
|
|
305
|
+
}): Promise<PathValue[]> {
|
|
306
|
+
let buffers = await PathValueController.nodes[config.nodeId].getValuesByPathAndTime(config.entries);
|
|
307
|
+
return await pathValueSerializer.deserialize(buffers);
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
public async getValuesByPathAndTime(entries: AuditSnapshotEntry[]): Promise<Buffer[]> {
|
|
311
|
+
let values: PathValue[] = [];
|
|
312
|
+
for (let entry of entries) {
|
|
313
|
+
let value = authorityStorage.getValueExactIgnoreInvalid(entry.path, entry.time);
|
|
314
|
+
if (!value) continue;
|
|
315
|
+
if (value.isTransparent) continue;
|
|
316
|
+
if (value.canGCValue) continue;
|
|
317
|
+
values.push(value);
|
|
318
|
+
}
|
|
319
|
+
let buffers = await pathValueSerializer.serialize(values, {
|
|
320
|
+
noLocks: true,
|
|
321
|
+
compress: getCompressNetwork(),
|
|
322
|
+
});
|
|
323
|
+
return buffers;
|
|
324
|
+
}
|
|
255
325
|
}
|
|
256
326
|
|
|
257
327
|
export const PathValueController = SocketFunction.register(
|
|
@@ -261,6 +331,8 @@ export const PathValueController = SocketFunction.register(
|
|
|
261
331
|
sendData: {},
|
|
262
332
|
|
|
263
333
|
getInitialValues: {},
|
|
334
|
+
getAuditSnapshot: {},
|
|
335
|
+
getValuesByPathAndTime: {},
|
|
264
336
|
|
|
265
337
|
watchLatest: {},
|
|
266
338
|
unwatchLatest: {},
|
|
@@ -271,5 +343,4 @@ export const PathValueController = SocketFunction.register(
|
|
|
271
343
|
{
|
|
272
344
|
noFunctionMeasure: !isNode(),
|
|
273
345
|
}
|
|
274
|
-
);
|
|
275
|
-
|
|
346
|
+
);
|
|
@@ -12,13 +12,43 @@ import { PathRouter } from "./PathRouter";
|
|
|
12
12
|
import { auditLog } from "./auditLogs";
|
|
13
13
|
import { WatchConfig, authorityStorage, PathValue, NodeId, compareTime, isCoreQuiet, MAX_CHANGE_AGE, createMissingEpochValue } from "./pathValueCore";
|
|
14
14
|
import { matchesParentRangeFilter } from "./hackedPackedPathParentFiltering";
|
|
15
|
+
import { isClient } from "../config2";
|
|
16
|
+
import { delay } from "socket-function/src/batching";
|
|
15
17
|
|
|
18
|
+
setImmediate(() => import("../4-querysub/Querysub"));
|
|
16
19
|
|
|
17
20
|
// These are used so we can have our internal trigger lookup without watching them on remote values.
|
|
18
21
|
export function getInternalValueWatchPrefix() {
|
|
19
22
|
return "INTERNAL_VALUE_WATCH_PREFIX_";
|
|
20
23
|
}
|
|
21
24
|
|
|
25
|
+
let pathWatcherState: (() => { watcherSequence: number }) | undefined;
|
|
26
|
+
|
|
27
|
+
setImmediate(async () => {
|
|
28
|
+
const { createLocalSchema } = await import("../4-querysub/schemaHelpers");
|
|
29
|
+
const { t } = await import("../2-proxy/schema2");
|
|
30
|
+
pathWatcherState = createLocalSchema("pathWatcherState", {
|
|
31
|
+
watcherSequence: t.number
|
|
32
|
+
});
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
let triggering = false;
|
|
36
|
+
function incrementWatcherSequence() {
|
|
37
|
+
if (!isClient()) return;
|
|
38
|
+
if (!pathWatcherState) return;
|
|
39
|
+
if (triggering) return;
|
|
40
|
+
triggering = true;
|
|
41
|
+
void delay(10).finally(async () => {
|
|
42
|
+
let { Querysub } = await import("../4-querysub/Querysub");
|
|
43
|
+
triggering = false;
|
|
44
|
+
Querysub.commit(() => {
|
|
45
|
+
if (!pathWatcherState) return;
|
|
46
|
+
let state = pathWatcherState();
|
|
47
|
+
state.watcherSequence = (state.watcherSequence || 0) + 1;
|
|
48
|
+
});
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
|
|
22
52
|
// WATCH CASES
|
|
23
53
|
// 1) getOwnNodeId() is used to trigger pathValueClientWatcher (no one else should be using it)
|
|
24
54
|
// 2) Use other nodes to immediately proxy to the other nodes
|
|
@@ -133,6 +163,7 @@ class PathWatcher {
|
|
|
133
163
|
}
|
|
134
164
|
|
|
135
165
|
if (newPathsWatched.size > 0 || newParentsWatched.size > 0) {
|
|
166
|
+
incrementWatcherSequence();
|
|
136
167
|
if (isOwnNodeId(config.nodeId)) {
|
|
137
168
|
for (let path of newPathsWatched) {
|
|
138
169
|
auditLog("new local WATCH VALUE", { path });
|
|
@@ -269,6 +300,7 @@ class PathWatcher {
|
|
|
269
300
|
}
|
|
270
301
|
|
|
271
302
|
if (fullyUnwatched.paths.length > 0 || fullyUnwatched.parentPaths.length > 0) {
|
|
303
|
+
incrementWatcherSequence();
|
|
272
304
|
console.info(`Unwatched PathValue watches`, {
|
|
273
305
|
unwatchedPaths: fullyUnwatched.paths.length,
|
|
274
306
|
unwatchedParents: fullyUnwatched.parentPaths.length,
|
|
@@ -505,6 +537,13 @@ class PathWatcher {
|
|
|
505
537
|
return Array.from(this.parentWatchers.keys());
|
|
506
538
|
}
|
|
507
539
|
|
|
540
|
+
public getAllWatchedPaths(): string[] {
|
|
541
|
+
if (pathWatcherState) {
|
|
542
|
+
pathWatcherState().watcherSequence;
|
|
543
|
+
}
|
|
544
|
+
return Array.from(this.watchers.keys());
|
|
545
|
+
}
|
|
546
|
+
|
|
508
547
|
public debug_harvestSyncTimes(): { start: number; end: number; path: string; }[] {
|
|
509
548
|
let times = this.syncHistoryForHarvest.getAllUnordered();
|
|
510
549
|
this.syncHistoryForHarvest.reset();
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
After a PathServerValue starts, it can't change its prefixes, as if the routing changes it makes it too difficult for clients to reason about who they should be synchronizing with.
|
|
10
10
|
*/
|
|
11
11
|
|
|
12
|
+
import { isNode } from "typesafecss";
|
|
12
13
|
import { nestArchives } from "../-a-archives/archives";
|
|
13
14
|
import { getArchivesBackblaze } from "../-a-archives/archivesBackBlaze";
|
|
14
15
|
import { archiveJSONT } from "../-a-archives/archivesJSONT";
|
|
@@ -23,6 +24,7 @@ let prefixes = archiveJSONT<PrefixObj>(() => nestArchives("shard-prefixes/", get
|
|
|
23
24
|
const key = "all";
|
|
24
25
|
|
|
25
26
|
export async function getShardPrefixes(): Promise<string[]> {
|
|
27
|
+
if (!isNode()) return [];
|
|
26
28
|
return (await prefixes.get(key))?.prefixes ?? [];
|
|
27
29
|
}
|
|
28
30
|
|
|
@@ -5,7 +5,7 @@ import { isDiskAudit } from "../config";
|
|
|
5
5
|
import { getPathFromStr } from "../path";
|
|
6
6
|
import { auditLog } from "./auditLogs";
|
|
7
7
|
import { PathRouter } from "./PathRouter";
|
|
8
|
-
import { PathValue, authorityStorage, compareTime, debugPathValuePath, ReadLock, byLockGroup, isCoreQuiet, debugRejections, debugTime, debugPathValue } from "./pathValueCore";
|
|
8
|
+
import { PathValue, authorityStorage, compareTime, debugPathValuePath, ReadLock, byLockGroup, isCoreQuiet, debugRejections, debugTime, debugPathValue, MAX_CHANGE_AGE } from "./pathValueCore";
|
|
9
9
|
import { pathWatcher } from "./PathWatcher";
|
|
10
10
|
import { lockWatcher2 } from "./LockWatcher2";
|
|
11
11
|
import { onPathInteracted } from "../diagnostics/pathAuditerCallback";
|
|
@@ -82,7 +82,9 @@ class ValidStateComputer {
|
|
|
82
82
|
console.info(`Rejecting past value due to initial sync: ${debugPathValuePath(value)}`, {
|
|
83
83
|
path: value.path,
|
|
84
84
|
timeId: value.time.time,
|
|
85
|
+
timeIdFull: value.time,
|
|
85
86
|
newTimeId: newValue?.time.time,
|
|
87
|
+
newTimeIdFull: newValue?.time,
|
|
86
88
|
});
|
|
87
89
|
ourValues.push({ ...value, valid: false, });
|
|
88
90
|
}
|
|
@@ -158,10 +160,11 @@ class ValidStateComputer {
|
|
|
158
160
|
}
|
|
159
161
|
return !!value.valid;
|
|
160
162
|
}
|
|
161
|
-
private isLockContentionFree(lock: ReadLock): PathValue | undefined {
|
|
163
|
+
private isLockContentionFree(lock: ReadLock): PathValue[] | undefined {
|
|
162
164
|
// sorted by -time (newest are first)
|
|
163
165
|
let history = authorityStorage.getValuePlusHistory(lock.path);
|
|
164
166
|
|
|
167
|
+
let conflictingValues: PathValue[] | undefined;
|
|
165
168
|
let index = binarySearchIndex(history.length, i => compareTime(lock.endTime, history[i].time));
|
|
166
169
|
if (index < 0) index = ~index;
|
|
167
170
|
else index++;
|
|
@@ -172,10 +175,11 @@ class ValidStateComputer {
|
|
|
172
175
|
if (value.isTransparent && lock.readIsTransparent) continue;
|
|
173
176
|
// If value.time is within our range, there's contention!
|
|
174
177
|
if (compareTime(lock.startTime, value.time) < 0 && compareTime(value.time, lock.endTime) < 0) {
|
|
175
|
-
|
|
178
|
+
if (!conflictingValues) conflictingValues = [];
|
|
179
|
+
conflictingValues.push(value);
|
|
176
180
|
}
|
|
177
181
|
}
|
|
178
|
-
return
|
|
182
|
+
return conflictingValues;
|
|
179
183
|
}
|
|
180
184
|
|
|
181
185
|
@measureFnc
|
|
@@ -193,6 +197,10 @@ class ValidStateComputer {
|
|
|
193
197
|
|
|
194
198
|
let valid = true;
|
|
195
199
|
for (let lock of locks) {
|
|
200
|
+
let age = now - lock.endTime.time;
|
|
201
|
+
// If it's old enough, then it's always valid, we can't change the valid save something after its max change age, even if it's invalid.
|
|
202
|
+
if (age > MAX_CHANGE_AGE) continue;
|
|
203
|
+
|
|
196
204
|
if (!this.isLockValid(lock)) {
|
|
197
205
|
onPathInteracted(lock.path, 2);
|
|
198
206
|
valid = false;
|
|
@@ -259,7 +267,7 @@ class ValidStateComputer {
|
|
|
259
267
|
}
|
|
260
268
|
}
|
|
261
269
|
}
|
|
262
|
-
private logLockContention(valueGroup: PathValue[], lock: ReadLock,
|
|
270
|
+
private logLockContention(valueGroup: PathValue[], lock: ReadLock, conflictingValues: PathValue[], now: number) {
|
|
263
271
|
|
|
264
272
|
// This special version indicates clientside prediction (which SHOULD be rejected).
|
|
265
273
|
if (lock.endTime.version !== Number.MAX_SAFE_INTEGER) {
|
|
@@ -278,7 +286,7 @@ class ValidStateComputer {
|
|
|
278
286
|
redMessage += `\n (original read from: ${debugTime(lock.startTime)}`;
|
|
279
287
|
redMessage += `\n (at time: ${debugTime(lock.endTime)}`;
|
|
280
288
|
redMessage += `\n (current time: ${Date.now()})`;
|
|
281
|
-
redMessage += `\n (conflict write at: ${debugTime(
|
|
289
|
+
redMessage += `\n (conflict write at: ${debugTime(conflictingValues[0].time)}`;
|
|
282
290
|
console.error(redMessage);
|
|
283
291
|
}
|
|
284
292
|
if (isDiskAudit()) {
|
|
@@ -287,11 +295,15 @@ class ValidStateComputer {
|
|
|
287
295
|
lock,
|
|
288
296
|
path: pathValue.path,
|
|
289
297
|
timeId: pathValue.time.time,
|
|
298
|
+
timeIdVersion: pathValue.time.version,
|
|
299
|
+
timeIdFull: pathValue.time,
|
|
290
300
|
lockedPath: lock.path,
|
|
291
301
|
transparent: lock.readIsTransparent,
|
|
292
302
|
missingPath: lock.path,
|
|
293
|
-
conflictTimeId0:
|
|
294
|
-
|
|
303
|
+
conflictTimeId0: conflictingValues[0].time.time,
|
|
304
|
+
conflictTimeIdVersion0: conflictingValues[0].time.version,
|
|
305
|
+
conflictCount: conflictingValues.length,
|
|
306
|
+
conflictingWrites: conflictingValues,
|
|
295
307
|
hadTimeId: lock.startTime.time,
|
|
296
308
|
});
|
|
297
309
|
}
|
|
@@ -1,22 +1,15 @@
|
|
|
1
1
|
import { measureWrap } from "socket-function/src/profiling/measure";
|
|
2
2
|
import { getPathDepth, getPathIndexAssert, hack_getPackedPathSuffix, hack_setPackedPathSuffix, hack_stripPackedPath } from "../path";
|
|
3
|
-
import { PathRouter } from "./PathRouter";
|
|
3
|
+
import { AuthoritySpec, PathRouter } from "./PathRouter";
|
|
4
4
|
import { cache } from "socket-function/src/caching";
|
|
5
5
|
import { authorityLookup } from "./AuthorityLookup";
|
|
6
6
|
|
|
7
|
-
// NOTE: This code has been moved into PathRouter
|
|
8
|
-
// /** Returns a number between 0 (inclusive) and 1 (exclusive)
|
|
9
|
-
// * - See matchesParentRangeFilter, matchesParentRangeFilterPart, and filterChildPaths.
|
|
10
|
-
// */
|
|
11
|
-
// export function __getRoutingHash(key: string): number {
|
|
12
|
-
// // Using fastHash is about twice as fast as sha256 (although that might be due to the buffer allocation?)
|
|
13
|
-
// let hash = fastHash(key);
|
|
14
|
-
// return hash % (1000 * 1000 * 1000) / (1000 * 1000 * 1000);
|
|
15
|
-
// // let hash = sha256(key);
|
|
16
|
-
// // return getBufferFraction(Buffer.from(hash, "hex"));
|
|
17
|
-
// }
|
|
18
7
|
|
|
19
|
-
|
|
8
|
+
let getSpecForChildPath = (path: string) => authorityLookup.getOurSpec();
|
|
9
|
+
export function registerGetSpecForChildPath(fnc: (path: string) => AuthoritySpec) {
|
|
10
|
+
getSpecForChildPath = fnc;
|
|
11
|
+
}
|
|
12
|
+
|
|
20
13
|
export function matchesParentRangeFilter(config: {
|
|
21
14
|
parentPath: string;
|
|
22
15
|
fullPath: string;
|
|
@@ -26,33 +19,22 @@ export function matchesParentRangeFilter(config: {
|
|
|
26
19
|
if (config.parentPath === config.packedPath) return true;
|
|
27
20
|
let filter = decodeParentFilter(config.packedPath);
|
|
28
21
|
if (!filter) return true;
|
|
29
|
-
|
|
22
|
+
if (filter.start <= 0 && filter.end >= 1) return true;
|
|
23
|
+
let route = PathRouter.getRouteFull({ path: config.fullPath, spec: getSpecForChildPath(config.fullPath) });
|
|
30
24
|
return filter.start <= route && route < filter.end;
|
|
31
25
|
}
|
|
32
|
-
// export function matchesParentRangeFilterPart(config: {
|
|
33
|
-
// part: string;
|
|
34
|
-
// start: number;
|
|
35
|
-
// end: number;
|
|
36
|
-
// }) {
|
|
37
|
-
// if (config.start === 0 && config.end === 1) return true;
|
|
38
|
-
// let hash = __getRoutingHash(config.part);
|
|
39
|
-
// return config.start <= hash && hash < config.end;
|
|
40
|
-
// }
|
|
41
26
|
|
|
42
|
-
// IMPORTANT! Child hashing is always based on the last key, never based on the full path. We need to make it always consistent, and we want it to work for parallel values. It cannot change depending on the routing, as this then becomes extremely complicated for proxies and for determining if we are actually supposed to be receiving a value from that node, and for the node to know what values it should send, etc.
|
|
43
27
|
export const filterChildPathsBase = measureWrap(
|
|
44
28
|
function filterChildPathsBase(parentPath: string, packedSuffix: string, paths: Set<string>): Set<string> {
|
|
45
29
|
let [startFractionStr, endFractionStr] = packedSuffix.split("|");
|
|
46
30
|
let startFraction = Number(startFractionStr);
|
|
47
31
|
let endFraction = Number(endFractionStr);
|
|
48
32
|
|
|
49
|
-
let depth = getPathDepth(parentPath);
|
|
50
|
-
|
|
51
33
|
let filtered = new Set<string>();
|
|
52
34
|
for (let path of paths) {
|
|
53
|
-
|
|
54
|
-
let
|
|
55
|
-
if (startFraction <=
|
|
35
|
+
// TODO: We can make this significantly more efficient as once we know if it's a prefix or not, we know the underlying function call every time. However, that starts to get complicated, and I don't think this is going to be a bottleneck.
|
|
36
|
+
let route = PathRouter.getRouteFull({ path, spec: getSpecForChildPath(path) });
|
|
37
|
+
if (startFraction <= route && route < endFraction) {
|
|
56
38
|
filtered.add(path);
|
|
57
39
|
}
|
|
58
40
|
}
|
|
@@ -223,6 +223,7 @@ export class PathValueArchives {
|
|
|
223
223
|
let totalSize = 0;
|
|
224
224
|
|
|
225
225
|
let dataPaths: string[] = [];
|
|
226
|
+
let allPaths: string[] = [];
|
|
226
227
|
let readCache = new Map<string, Buffer>();
|
|
227
228
|
|
|
228
229
|
let downloadStartTime = Date.now();
|
|
@@ -233,8 +234,12 @@ export class PathValueArchives {
|
|
|
233
234
|
// the source file, but by now it has moved, so we don't see either file, even though at all times
|
|
234
235
|
// both files existed).
|
|
235
236
|
let time = Date.now();
|
|
236
|
-
|
|
237
|
-
|
|
237
|
+
let valueObjs = await this.getValuePaths(authority);
|
|
238
|
+
dataPaths = valueObjs.pickedPaths;
|
|
239
|
+
allPaths = valueObjs.allPaths;
|
|
240
|
+
let allValueCount = allPaths.reduce((acc, path) => acc + this.decodeDataPath(path).valueCount, 0);
|
|
241
|
+
let pickedValueCount = dataPaths.reduce((acc, path) => acc + this.decodeDataPath(path).valueCount, 0);
|
|
242
|
+
console.log(green(`${dataPaths.length}/${allPaths.length} (values ${formatNumber(pickedValueCount)} / ${formatNumber(allValueCount)}) data paths in ${formatTime(Date.now() - time)}`));
|
|
238
243
|
// NOTE: If the notMatched count is high enough, it is possible NodePathAuthorities.ts:authoritiesMightOverlap is
|
|
239
244
|
// too loose, and should be removing more cases.
|
|
240
245
|
|
|
@@ -437,7 +442,10 @@ export class PathValueArchives {
|
|
|
437
442
|
}
|
|
438
443
|
|
|
439
444
|
@measureFnc
|
|
440
|
-
public async getValuePaths(authority: AuthoritySpec): Promise<
|
|
445
|
+
public async getValuePaths(authority: AuthoritySpec): Promise<{
|
|
446
|
+
pickedPaths: string[];
|
|
447
|
+
allPaths: string[];
|
|
448
|
+
}> {
|
|
441
449
|
// let paths: string[] = [];
|
|
442
450
|
// let pickedDirectories = await this.getAuthorityDirs(authority);
|
|
443
451
|
|
|
@@ -450,8 +458,11 @@ export class PathValueArchives {
|
|
|
450
458
|
|
|
451
459
|
let locker = await this.getArchiveLocker();
|
|
452
460
|
let allFiles = (await locker.getAllValidFiles()).map(x => x.file);
|
|
453
|
-
let
|
|
454
|
-
return
|
|
461
|
+
let pickedPaths = allFiles.filter(x => PathRouter.overlapsPathIdentifier(authority, x));
|
|
462
|
+
return {
|
|
463
|
+
pickedPaths,
|
|
464
|
+
allPaths: allFiles,
|
|
465
|
+
};
|
|
455
466
|
}
|
|
456
467
|
@measureFnc
|
|
457
468
|
public async getValuePathSizes(paths: string[]): Promise<number[]> {
|
|
@@ -24,6 +24,8 @@ import { fastHash } from "../misc/hash";
|
|
|
24
24
|
import { authorityLookup } from "./AuthorityLookup";
|
|
25
25
|
import { onPathInteracted } from "../diagnostics/pathAuditerCallback";
|
|
26
26
|
import { decodeParentFilter, filterChildPathsBase } from "./hackedPackedPathParentFiltering";
|
|
27
|
+
import { isDiskAudit } from "../config";
|
|
28
|
+
import { removeRange } from "../rangeMath";
|
|
27
29
|
|
|
28
30
|
setImmediate(async () => {
|
|
29
31
|
// Import everything will dynamically import, so the client side can tell that it's required.
|
|
@@ -543,6 +545,9 @@ class AuthorityPathValueStorage {
|
|
|
543
545
|
if (this.DEBUG_UNWATCH) {
|
|
544
546
|
console.log(blue(`Unsyncing path at ${Date.now()}`), path);
|
|
545
547
|
}
|
|
548
|
+
if (isDiskAudit()) {
|
|
549
|
+
auditLog("DESTROY PATH", { path });
|
|
550
|
+
}
|
|
546
551
|
|
|
547
552
|
this.isSyncedCache.delete(path);
|
|
548
553
|
this.removePathFromStorage(path, "unwatched");
|
|
@@ -604,11 +609,45 @@ class AuthorityPathValueStorage {
|
|
|
604
609
|
return false;
|
|
605
610
|
}
|
|
606
611
|
public isParentSynced(path: string) {
|
|
612
|
+
let range = decodeParentFilter(path);
|
|
607
613
|
path = hack_stripPackedPath(path);
|
|
608
|
-
|
|
609
|
-
|
|
614
|
+
if (PathRouter.isLocalPath(path)) return true;
|
|
615
|
+
|
|
616
|
+
let synced = this.parentsSynced.get(path);
|
|
617
|
+
if (synced === true) return true;
|
|
618
|
+
// See if the ranges received so far cover the requested range. If we only request a partial range, then this will always be the case. We'll never fully synchronize the path.
|
|
619
|
+
if (synced && range) {
|
|
620
|
+
let missingRanges: { start: number; end: number }[] = [{ start: range.start, end: range.end }];
|
|
621
|
+
for (let range of synced) {
|
|
622
|
+
removeRange(missingRanges, range);
|
|
623
|
+
}
|
|
624
|
+
if (missingRanges.length === 0) return true;
|
|
625
|
+
}
|
|
610
626
|
|
|
611
|
-
|
|
627
|
+
// Self authority check
|
|
628
|
+
{
|
|
629
|
+
let nodes = PathRouter.getChildReadNodes(path, { onlyOwnNodes: true });
|
|
630
|
+
if (nodes.nodes.length > 0) {
|
|
631
|
+
if (nodes.nodes.every(x => isOwnNodeId(x.nodeId))) {
|
|
632
|
+
// If it's not a partial 30 and we're self authority, we're always going to be self authority, so we can just cache it in the lookup.
|
|
633
|
+
this.parentsSynced.set(path, true);
|
|
634
|
+
return true;
|
|
635
|
+
}
|
|
636
|
+
// Might be a partial authority
|
|
637
|
+
if (range) {
|
|
638
|
+
let selfNodes = nodes.nodes.filter(x => isOwnNodeId(x.nodeId));
|
|
639
|
+
let missingRanges: { start: number; end: number }[] = [{ start: range.start, end: range.end }];
|
|
640
|
+
for (let range of selfNodes) {
|
|
641
|
+
removeRange(missingRanges, range.range);
|
|
642
|
+
}
|
|
643
|
+
if (missingRanges.length === 0) {
|
|
644
|
+
return true;
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
}
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
return false;
|
|
612
651
|
}
|
|
613
652
|
|
|
614
653
|
/** Obviously just for debugging. Doesn't trigger any watchers, just erases it as if we never had it. There shouldn't be really any downstream caches within the same process at least, and so this should result in the next read having the value missing. If we in the future add downstream caches, we're going to have to invalidate them as well.
|
|
@@ -1052,6 +1091,7 @@ class AuthorityPathValueStorage {
|
|
|
1052
1091
|
let { spec, startTime, endTime } = config;
|
|
1053
1092
|
let result: PathValue[] = [];
|
|
1054
1093
|
for (let [path, values] of this.values) {
|
|
1094
|
+
if (PathRouter.isLocalPath(path)) continue;
|
|
1055
1095
|
if (PathRouter.matchesAuthoritySpec(spec, path)) {
|
|
1056
1096
|
for (let value of values) {
|
|
1057
1097
|
let time = value.time.time;
|