querysub 0.402.0 → 0.404.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/.cursorrules +2 -0
  2. package/bin/audit-imports.js +4 -0
  3. package/package.json +7 -4
  4. package/spec.txt +77 -0
  5. package/src/-a-archives/archiveCache.ts +9 -4
  6. package/src/-a-archives/archivesBackBlaze.ts +1039 -1039
  7. package/src/-a-auth/certs.ts +0 -12
  8. package/src/-c-identity/IdentityController.ts +12 -3
  9. package/src/-f-node-discovery/NodeDiscovery.ts +32 -26
  10. package/src/-g-core-values/NodeCapabilities.ts +12 -2
  11. package/src/0-path-value-core/AuthorityLookup.ts +239 -0
  12. package/src/0-path-value-core/LockWatcher2.ts +150 -0
  13. package/src/0-path-value-core/PathRouter.ts +535 -0
  14. package/src/0-path-value-core/PathRouterRouteOverride.ts +72 -0
  15. package/src/0-path-value-core/PathRouterServerAuthoritySpec.tsx +65 -0
  16. package/src/0-path-value-core/PathValueCommitter.ts +222 -488
  17. package/src/0-path-value-core/PathValueController.ts +277 -239
  18. package/src/0-path-value-core/PathWatcher.ts +534 -0
  19. package/src/0-path-value-core/ShardPrefixes.ts +31 -0
  20. package/src/0-path-value-core/ValidStateComputer.ts +303 -0
  21. package/src/0-path-value-core/archiveLocks/ArchiveLocks.ts +1 -1
  22. package/src/0-path-value-core/archiveLocks/ArchiveLocks2.ts +80 -44
  23. package/src/0-path-value-core/archiveLocks/archiveSnapshots.ts +13 -16
  24. package/src/0-path-value-core/auditLogs.ts +2 -0
  25. package/src/0-path-value-core/hackedPackedPathParentFiltering.ts +97 -0
  26. package/src/0-path-value-core/pathValueArchives.ts +490 -492
  27. package/src/0-path-value-core/pathValueCore.ts +195 -1492
  28. package/src/0-path-value-core/startupAuthority.ts +74 -0
  29. package/src/1-path-client/RemoteWatcher.ts +100 -83
  30. package/src/1-path-client/pathValueClientWatcher.ts +808 -815
  31. package/src/2-proxy/PathValueProxyWatcher.ts +10 -8
  32. package/src/2-proxy/archiveMoveHarness.ts +182 -214
  33. package/src/2-proxy/garbageCollection.ts +9 -8
  34. package/src/2-proxy/schema2.ts +21 -1
  35. package/src/3-path-functions/PathFunctionHelpers.ts +206 -180
  36. package/src/3-path-functions/PathFunctionRunner.ts +943 -766
  37. package/src/3-path-functions/PathFunctionRunnerMain.ts +5 -3
  38. package/src/3-path-functions/pathFunctionLoader.ts +2 -2
  39. package/src/3-path-functions/syncSchema.ts +592 -521
  40. package/src/4-deploy/deployFunctions.ts +19 -4
  41. package/src/4-deploy/deployGetFunctionsInner.ts +8 -2
  42. package/src/4-deploy/deployMain.ts +51 -68
  43. package/src/4-deploy/edgeClientWatcher.tsx +1 -1
  44. package/src/4-deploy/edgeNodes.ts +2 -2
  45. package/src/4-dom/qreact.tsx +2 -4
  46. package/src/4-dom/qreactTest.tsx +7 -13
  47. package/src/4-querysub/Querysub.ts +21 -8
  48. package/src/4-querysub/QuerysubController.ts +45 -29
  49. package/src/4-querysub/permissions.ts +2 -2
  50. package/src/4-querysub/querysubPrediction.ts +80 -70
  51. package/src/4-querysub/schemaHelpers.ts +5 -1
  52. package/src/5-diagnostics/GenericFormat.tsx +14 -9
  53. package/src/archiveapps/archiveGCEntry.tsx +9 -2
  54. package/src/archiveapps/archiveJoinEntry.ts +87 -84
  55. package/src/archiveapps/archiveMergeEntry.tsx +2 -0
  56. package/src/bits.ts +19 -0
  57. package/src/config.ts +21 -3
  58. package/src/config2.ts +23 -48
  59. package/src/deployManager/components/DeployPage.tsx +7 -3
  60. package/src/deployManager/machineSchema.ts +4 -1
  61. package/src/diagnostics/ActionsHistory.ts +3 -8
  62. package/src/diagnostics/AuditLogPage.tsx +2 -3
  63. package/src/diagnostics/FunctionCallInfo.tsx +141 -0
  64. package/src/diagnostics/FunctionCallInfoState.ts +162 -0
  65. package/src/diagnostics/MachineThreadInfo.tsx +1 -1
  66. package/src/diagnostics/NodeViewer.tsx +37 -48
  67. package/src/diagnostics/SyncTestPage.tsx +241 -0
  68. package/src/diagnostics/auditImportViolations.ts +185 -0
  69. package/src/diagnostics/listenOnDebugger.ts +3 -3
  70. package/src/diagnostics/logs/IndexedLogs/BufferUnitSet.ts +10 -4
  71. package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts +2 -2
  72. package/src/diagnostics/logs/IndexedLogs/LogViewer3.tsx +24 -22
  73. package/src/diagnostics/logs/IndexedLogs/moveIndexLogsToPublic.ts +1 -1
  74. package/src/diagnostics/logs/diskLogGlobalContext.ts +1 -0
  75. package/src/diagnostics/logs/errorNotifications2/logWatcher.ts +1 -3
  76. package/src/diagnostics/logs/lifeCycleAnalysis/LifeCycleEntryEditor.tsx +39 -17
  77. package/src/diagnostics/logs/lifeCycleAnalysis/LifeCycleEntryReadMode.tsx +4 -6
  78. package/src/diagnostics/logs/lifeCycleAnalysis/LifeCycleInstanceTableView.tsx +36 -5
  79. package/src/diagnostics/logs/lifeCycleAnalysis/LifeCyclePage.tsx +19 -5
  80. package/src/diagnostics/logs/lifeCycleAnalysis/LifeCycleRenderer.tsx +15 -7
  81. package/src/diagnostics/logs/lifeCycleAnalysis/NestedLifeCycleInfo.tsx +28 -106
  82. package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycleMatching.ts +2 -0
  83. package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycleMisc.ts +0 -0
  84. package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycleSearch.tsx +18 -7
  85. package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycles.tsx +3 -0
  86. package/src/diagnostics/managementPages.tsx +10 -3
  87. package/src/diagnostics/misc-pages/ArchiveViewer.tsx +20 -26
  88. package/src/diagnostics/misc-pages/ArchiveViewerTree.tsx +6 -4
  89. package/src/diagnostics/misc-pages/ComponentSyncStats.tsx +2 -2
  90. package/src/diagnostics/misc-pages/LocalWatchViewer.tsx +7 -9
  91. package/src/diagnostics/misc-pages/SnapshotViewer.tsx +23 -12
  92. package/src/diagnostics/misc-pages/archiveViewerShared.tsx +1 -1
  93. package/src/diagnostics/pathAuditer.ts +486 -0
  94. package/src/diagnostics/pathAuditerCallback.ts +20 -0
  95. package/src/diagnostics/watchdog.ts +8 -1
  96. package/src/library-components/URLParam.ts +1 -1
  97. package/src/misc/hash.ts +1 -0
  98. package/src/path.ts +21 -7
  99. package/src/server.ts +54 -47
  100. package/src/user-implementation/loginEmail.tsx +1 -1
  101. package/tempnotes.txt +67 -0
  102. package/test.ts +288 -95
  103. package/src/0-path-value-core/NodePathAuthorities.ts +0 -1057
  104. package/src/0-path-value-core/PathController.ts +0 -1
  105. package/src/5-diagnostics/diskValueAudit.ts +0 -218
  106. package/src/5-diagnostics/memoryValueAudit.ts +0 -438
  107. package/src/archiveapps/lockTest.ts +0 -127
@@ -1,127 +0,0 @@
1
- import { pathValueArchives } from "../0-path-value-core/pathValueArchives";
2
- import { getPathStr, rootPathStr } from "../path";
3
- import { ArchiveLocker, ArchiveTransaction } from "../0-path-value-core/archiveLocks/ArchiveLocks";
4
- import { getDomain } from "../config";
5
- import { getOurAuthorities } from "../config2";
6
- import { getNodeId } from "socket-function/src/nodeCache";
7
- import { getOurNodeIdAssert, getOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
8
- import { Querysub } from "../4-querysub/QuerysubController";
9
- import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
10
- import { AuthorityPath, pathValueAuthority2 } from "../0-path-value-core/NodePathAuthorities";
11
- import { PathValue, getNextTime } from "../0-path-value-core/pathValueCore";
12
- import debugbreak from "debugbreak";
13
- import { delay } from "socket-function/src/batching";
14
- import { isDefined } from "../misc";
15
- import { sort, timeInMinute } from "socket-function/src/misc";
16
- import { green, red } from "socket-function/src/formatting/logColors";
17
- import { formatTime } from "socket-function/src/formatting/format";
18
-
19
- // yarn typenode src\framework-beta\archivemerge\lockTest.ts --nonetwork
20
- async function main() {
21
- await Querysub.hostService("lockTest");
22
- /*
23
- let allLockers: ArchiveLocker[] = [];
24
- for (let authority of getOurAuthorities()) {
25
- let lockers = await pathValueArchives.getRawArchives(authority);
26
- allLockers.push(...lockers);
27
- }
28
- */
29
-
30
- const START = 5;
31
- const ITERATION_COUNT = 5;
32
- const COUNT_LIMIT = ITERATION_COUNT * 5;
33
-
34
- // D:\repos\qs-aipaint\database-storage\archives\path-values-recycle-bin\=-.,querysub.__com.,
35
-
36
- let authority: AuthorityPath = getOurAuthorities()[0];
37
- let locker = await pathValueArchives.getArchiveLocker();
38
- let matchedDirs = await pathValueArchives.getAuthorityDirs(authority);
39
- const authorityDir = pathValueAuthority2.getArchiveDirectory(authority);
40
- while (true) {
41
- let result = await locker.atomicSwapFiles({}, async (valueFiles, readFiles) => {
42
- valueFiles = valueFiles.filter(x => matchedDirs.some(y => x.file.startsWith(y)));
43
- let prevFiles = valueFiles.filter(x => pathValueArchives.decodeDataPath(x.file).sourceType === "test");
44
- let prevData = (await readFiles(prevFiles)).filter(isDefined);
45
-
46
- let pathValues: number[][] = [];
47
- for (let prevBuffer of prevData) {
48
- let prevValue = await pathValueSerializer.deserialize([prevBuffer], { singleBuffer: true });
49
- pathValues.push(prevValue.map(x => pathValueSerializer.getPathValue(x) as number));
50
- }
51
- console.log(`Values [${pathValues.map(x => "[" + x.join(", ") + "]").join(", ")}]`);
52
-
53
- let flatValues = pathValues.flat();
54
- sort(flatValues, x => x);
55
- let minValue = flatValues[0] || START;
56
- if (flatValues.length > 0) {
57
- let expectedValues = Array.from({ length: flatValues.length }, (_, i) => i + minValue);
58
- if (flatValues.join(",") !== expectedValues.join(",")) {
59
- const errorMessage = `Values should be [${expectedValues.join(", ")}], was [${flatValues.join(", ")}]`;
60
- console.error(errorMessage);
61
- debugbreak(2);
62
- debugger;
63
- throw new Error(errorMessage);
64
- }
65
- }
66
-
67
- const transaction: ArchiveTransaction = {
68
- createFiles: [],
69
- deleteFiles: [],
70
- };
71
- function removeValue(value: number) {
72
- console.log(`Deleting file with value ${value}`);
73
- let index = pathValues.findIndex(x => x[0] === value);
74
- let file = prevFiles[index];
75
- transaction.deleteFiles.push(file);
76
- }
77
- async function addValue(newValue: number) {
78
- console.log(`Adding file with value ${newValue}`);
79
- let newValues: PathValue[] = [];
80
- newValues.push({
81
- path: getPathStr([getDomain(), "test"]),
82
- value: newValue,
83
- time: getNextTime(),
84
- locks: [],
85
- lockCount: 0,
86
- valid: true,
87
- });
88
- const dataObj = (await pathValueArchives.encodeValuePaths(
89
- newValues,
90
- {
91
- pathOverrides: {
92
- sourceType: "test",
93
- misc: newValue + ""
94
- },
95
- }
96
- ))!;
97
- transaction.createFiles.push({
98
- data: dataObj.data,
99
- file: authorityDir + dataObj.file,
100
- });
101
- }
102
-
103
- let newCount = flatValues.length + ITERATION_COUNT;
104
- let extraCount = newCount - COUNT_LIMIT;
105
- for (let i = 0; i < extraCount; i++) {
106
- removeValue(flatValues[i]);
107
- }
108
-
109
- let lastValue = flatValues[flatValues.length - 1] ?? START;
110
- for (let i = 0; i < ITERATION_COUNT; i++) {
111
- await addValue(lastValue + i + 1);
112
- }
113
-
114
- let timeWaitTime = Math.random() * 5 * 1000;
115
- console.log(`Test wait ${formatTime(timeWaitTime)}`);
116
- await delay(timeWaitTime);
117
-
118
- return [transaction];
119
- });
120
- if (result === "rejected") {
121
- console.warn(red("Rejected transaction"));
122
- } else {
123
- console.log(green("Transaction committed"));
124
- }
125
- }
126
- }
127
- main().catch(console.error).finally(() => process.exit());