querysub 0.402.0 → 0.404.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/.cursorrules +2 -0
  2. package/bin/audit-imports.js +4 -0
  3. package/package.json +7 -4
  4. package/spec.txt +77 -0
  5. package/src/-a-archives/archiveCache.ts +9 -4
  6. package/src/-a-archives/archivesBackBlaze.ts +1039 -1039
  7. package/src/-a-auth/certs.ts +0 -12
  8. package/src/-c-identity/IdentityController.ts +12 -3
  9. package/src/-f-node-discovery/NodeDiscovery.ts +32 -26
  10. package/src/-g-core-values/NodeCapabilities.ts +12 -2
  11. package/src/0-path-value-core/AuthorityLookup.ts +239 -0
  12. package/src/0-path-value-core/LockWatcher2.ts +150 -0
  13. package/src/0-path-value-core/PathRouter.ts +535 -0
  14. package/src/0-path-value-core/PathRouterRouteOverride.ts +72 -0
  15. package/src/0-path-value-core/PathRouterServerAuthoritySpec.tsx +65 -0
  16. package/src/0-path-value-core/PathValueCommitter.ts +222 -488
  17. package/src/0-path-value-core/PathValueController.ts +277 -239
  18. package/src/0-path-value-core/PathWatcher.ts +534 -0
  19. package/src/0-path-value-core/ShardPrefixes.ts +31 -0
  20. package/src/0-path-value-core/ValidStateComputer.ts +303 -0
  21. package/src/0-path-value-core/archiveLocks/ArchiveLocks.ts +1 -1
  22. package/src/0-path-value-core/archiveLocks/ArchiveLocks2.ts +80 -44
  23. package/src/0-path-value-core/archiveLocks/archiveSnapshots.ts +13 -16
  24. package/src/0-path-value-core/auditLogs.ts +2 -0
  25. package/src/0-path-value-core/hackedPackedPathParentFiltering.ts +97 -0
  26. package/src/0-path-value-core/pathValueArchives.ts +490 -492
  27. package/src/0-path-value-core/pathValueCore.ts +195 -1492
  28. package/src/0-path-value-core/startupAuthority.ts +74 -0
  29. package/src/1-path-client/RemoteWatcher.ts +100 -83
  30. package/src/1-path-client/pathValueClientWatcher.ts +808 -815
  31. package/src/2-proxy/PathValueProxyWatcher.ts +10 -8
  32. package/src/2-proxy/archiveMoveHarness.ts +182 -214
  33. package/src/2-proxy/garbageCollection.ts +9 -8
  34. package/src/2-proxy/schema2.ts +21 -1
  35. package/src/3-path-functions/PathFunctionHelpers.ts +206 -180
  36. package/src/3-path-functions/PathFunctionRunner.ts +943 -766
  37. package/src/3-path-functions/PathFunctionRunnerMain.ts +5 -3
  38. package/src/3-path-functions/pathFunctionLoader.ts +2 -2
  39. package/src/3-path-functions/syncSchema.ts +592 -521
  40. package/src/4-deploy/deployFunctions.ts +19 -4
  41. package/src/4-deploy/deployGetFunctionsInner.ts +8 -2
  42. package/src/4-deploy/deployMain.ts +51 -68
  43. package/src/4-deploy/edgeClientWatcher.tsx +1 -1
  44. package/src/4-deploy/edgeNodes.ts +2 -2
  45. package/src/4-dom/qreact.tsx +2 -4
  46. package/src/4-dom/qreactTest.tsx +7 -13
  47. package/src/4-querysub/Querysub.ts +21 -8
  48. package/src/4-querysub/QuerysubController.ts +45 -29
  49. package/src/4-querysub/permissions.ts +2 -2
  50. package/src/4-querysub/querysubPrediction.ts +80 -70
  51. package/src/4-querysub/schemaHelpers.ts +5 -1
  52. package/src/5-diagnostics/GenericFormat.tsx +14 -9
  53. package/src/archiveapps/archiveGCEntry.tsx +9 -2
  54. package/src/archiveapps/archiveJoinEntry.ts +87 -84
  55. package/src/archiveapps/archiveMergeEntry.tsx +2 -0
  56. package/src/bits.ts +19 -0
  57. package/src/config.ts +21 -3
  58. package/src/config2.ts +23 -48
  59. package/src/deployManager/components/DeployPage.tsx +7 -3
  60. package/src/deployManager/machineSchema.ts +4 -1
  61. package/src/diagnostics/ActionsHistory.ts +3 -8
  62. package/src/diagnostics/AuditLogPage.tsx +2 -3
  63. package/src/diagnostics/FunctionCallInfo.tsx +141 -0
  64. package/src/diagnostics/FunctionCallInfoState.ts +162 -0
  65. package/src/diagnostics/MachineThreadInfo.tsx +1 -1
  66. package/src/diagnostics/NodeViewer.tsx +37 -48
  67. package/src/diagnostics/SyncTestPage.tsx +241 -0
  68. package/src/diagnostics/auditImportViolations.ts +185 -0
  69. package/src/diagnostics/listenOnDebugger.ts +3 -3
  70. package/src/diagnostics/logs/IndexedLogs/BufferUnitSet.ts +10 -4
  71. package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts +2 -2
  72. package/src/diagnostics/logs/IndexedLogs/LogViewer3.tsx +24 -22
  73. package/src/diagnostics/logs/IndexedLogs/moveIndexLogsToPublic.ts +1 -1
  74. package/src/diagnostics/logs/diskLogGlobalContext.ts +1 -0
  75. package/src/diagnostics/logs/errorNotifications2/logWatcher.ts +1 -3
  76. package/src/diagnostics/logs/lifeCycleAnalysis/LifeCycleEntryEditor.tsx +39 -17
  77. package/src/diagnostics/logs/lifeCycleAnalysis/LifeCycleEntryReadMode.tsx +4 -6
  78. package/src/diagnostics/logs/lifeCycleAnalysis/LifeCycleInstanceTableView.tsx +36 -5
  79. package/src/diagnostics/logs/lifeCycleAnalysis/LifeCyclePage.tsx +19 -5
  80. package/src/diagnostics/logs/lifeCycleAnalysis/LifeCycleRenderer.tsx +15 -7
  81. package/src/diagnostics/logs/lifeCycleAnalysis/NestedLifeCycleInfo.tsx +28 -106
  82. package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycleMatching.ts +2 -0
  83. package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycleMisc.ts +0 -0
  84. package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycleSearch.tsx +18 -7
  85. package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycles.tsx +3 -0
  86. package/src/diagnostics/managementPages.tsx +10 -3
  87. package/src/diagnostics/misc-pages/ArchiveViewer.tsx +20 -26
  88. package/src/diagnostics/misc-pages/ArchiveViewerTree.tsx +6 -4
  89. package/src/diagnostics/misc-pages/ComponentSyncStats.tsx +2 -2
  90. package/src/diagnostics/misc-pages/LocalWatchViewer.tsx +7 -9
  91. package/src/diagnostics/misc-pages/SnapshotViewer.tsx +23 -12
  92. package/src/diagnostics/misc-pages/archiveViewerShared.tsx +1 -1
  93. package/src/diagnostics/pathAuditer.ts +486 -0
  94. package/src/diagnostics/pathAuditerCallback.ts +20 -0
  95. package/src/diagnostics/watchdog.ts +8 -1
  96. package/src/library-components/URLParam.ts +1 -1
  97. package/src/misc/hash.ts +1 -0
  98. package/src/path.ts +21 -7
  99. package/src/server.ts +54 -47
  100. package/src/user-implementation/loginEmail.tsx +1 -1
  101. package/tempnotes.txt +67 -0
  102. package/test.ts +288 -95
  103. package/src/0-path-value-core/NodePathAuthorities.ts +0 -1057
  104. package/src/0-path-value-core/PathController.ts +0 -1
  105. package/src/5-diagnostics/diskValueAudit.ts +0 -218
  106. package/src/5-diagnostics/memoryValueAudit.ts +0 -438
  107. package/src/archiveapps/lockTest.ts +0 -127
@@ -1,55 +1,42 @@
1
1
  import { SocketFunction } from "socket-function/SocketFunction";
2
2
  import { css } from "typesafecss";
3
- import { t } from "../../../2-proxy/schema2";
4
3
  import { qreact } from "../../../4-dom/qreact";
5
- import { formatValue } from "../../../5-diagnostics/GenericFormat";
6
- import { Button } from "../../../library-components/Button";
7
- import { niceStringify } from "../../../niceStringify";
8
- import { LogDatum } from "../diskLogger";
9
- import { LifeCycle, LifeCyclesController, getVariables } from "./lifeCycles";
10
- import { createLifeCycleSearch, LifecycleInstance } from "./lifeCycleSearch";
11
- import { IndexedLogResults } from "../IndexedLogs/BufferIndexHelpers";
4
+ import { LifeCycle, LifeCycleEntry, LifeCyclesController } from "./lifeCycles";
12
5
  import { formatSearchString } from "../IndexedLogs/LogViewerParams";
13
- import { getPathStr, getPathStr2 } from "../../../path";
6
+ import { ATag } from "../../../library-components/ATag";
7
+ import { lifecycleIdURL, limitURL, additionalSearchURL, phase2AdditionalSearchURL } from "./LifeCyclePage";
14
8
 
15
9
  export class NestedLifeCycleInfo extends qreact.Component<{
16
10
  keyName: string;
17
11
  value: unknown;
18
- aliases: string[];
12
+ entry: LifeCycleEntry;
19
13
  currentLifeCycleId: string;
20
14
  }> {
21
- state = t.state({
22
- searchingLifeCycleId: t.atomic<string | undefined>(undefined),
23
- phase1Results: t.atomic<LogDatum[]>([]),
24
- phase1InvalidResults: t.atomic<Array<{ datum: LogDatum; missingKeys: string[] }>>([]),
25
- phase1Stats: t.atomic<IndexedLogResults | undefined>(undefined),
26
- phase1Searching: t.boolean(false),
27
- phase2Results: t.atomic<LogDatum[]>([]),
28
- phase2Stats: t.atomic<IndexedLogResults | undefined>(undefined),
29
- phase2Searching: t.boolean(false),
30
- phase2HitLimit: t.boolean(false),
31
- lifecycleInstances: t.atomic<LifecycleInstance[]>([]),
32
- });
33
-
34
15
  controller = LifeCyclesController(SocketFunction.browserNodeId());
35
- search = createLifeCycleSearch(this);
36
16
 
37
17
  render() {
38
18
  let lifeCycles = this.controller.getLifeCycles();
39
19
  if (!lifeCycles) return null;
40
20
 
41
- let { keyName, value, aliases, currentLifeCycleId } = this.props;
21
+ let { keyName, value, entry, currentLifeCycleId } = this.props;
22
+
23
+ let isGroupByKey = Array.isArray(entry.groupByKeys) && entry.groupByKeys.some(k => k.ourKey === keyName);
24
+ let aliases = isGroupByKey && entry.groupByKeys.find(k => k.ourKey === keyName)?.aliases || entry.variables[keyName]?.aliases || [];
42
25
 
43
26
  let allKeysToMatch = [keyName, ...aliases].filter(x => !["__threadId", "__machineId"].includes(x));
44
27
  let matchingLifeCycles: Array<{ lifecycle: LifeCycle; matchedKey: string }> = [];
45
28
 
46
29
  for (let lifecycle of lifeCycles) {
47
- if (lifecycle.id === currentLifeCycleId) continue;
30
+ let keySources = allKeysToMatch;
31
+ // Don't match our keyName if we are the same lifecycle
32
+ if (lifecycle.id === currentLifeCycleId) {
33
+ keySources = aliases;
34
+ }
48
35
 
49
36
  let matchedKey: string | undefined = undefined;
50
37
  for (let entry of lifecycle.entries) {
51
38
  for (let groupByKey of entry.groupByKeys) {
52
- if (allKeysToMatch.includes(groupByKey.ourKey)) {
39
+ if (keySources.includes(groupByKey.ourKey)) {
53
40
  matchedKey = groupByKey.ourKey;
54
41
  break;
55
42
  }
@@ -66,85 +53,20 @@ export class NestedLifeCycleInfo extends qreact.Component<{
66
53
 
67
54
  return <>
68
55
  {matchingLifeCycles.map(({ lifecycle, matchedKey }) => {
69
- let isSearchingThis = this.state.searchingLifeCycleId === lifecycle.id;
70
- let isSearching = isSearchingThis && (this.state.phase1Searching || this.state.phase2Searching);
71
- let hasSearched = isSearchingThis && !isSearching;
72
-
73
- let instances = hasSearched && this.state.lifecycleInstances.filter(inst =>
74
- inst.keys.some(k => k.key === matchedKey && k.value === value)
75
- ) || [];
76
- let hasResults = instances.length > 0;
77
-
78
- let progressText = "";
79
- if (isSearching && isSearchingThis) {
80
- let parts: string[] = [];
81
-
82
- if (this.state.phase1Searching || this.state.phase1Stats) {
83
- let stats = this.state.phase1Stats;
84
- let blocksScanned = stats && (stats.localFilesSearched + stats.backblazeFilesSearched) || 0;
85
- let resultsCount = this.state.phase1Results.length;
86
- parts.push(`${blocksScanned} > ${resultsCount}`);
87
- }
88
-
89
- if (this.state.phase2Searching) {
90
- let stats = this.state.phase2Stats;
91
- let blocksScanned = stats && (stats.localFilesSearched + stats.backblazeFilesSearched) || 0;
92
- let resultsCount = this.state.phase2Results.length;
93
- parts.push(`${blocksScanned} > ${resultsCount}`);
94
- }
95
-
96
- progressText = parts.length > 0 && `(${parts.join(" | ")})` || "(...)";
97
- }
98
-
99
- return <>
100
- <Button
101
- hue={180}
102
- onClick={() => {
103
- void this.search.searchLifeCycle({
104
- lifeCycleId: lifecycle.id,
105
- limit: 10,
106
- additionalSearch: formatSearchString({ [matchedKey]: value })
107
- });
108
- }}
109
- >
110
- {lifecycle.title} {isSearching && "⏳" || ""} {progressText} {hasSearched && `(${instances.length})` || ""}
111
- </Button>
112
- {hasSearched && !hasResults && (
113
- <div className={css.pad2(4).colorhsl(0, 0, 50)}>No results found</div>
114
- )}
115
- {hasResults && (() => {
116
- let firstInstance = instances[0];
117
- let totalCount = instances.length;
118
- let hitLimit = totalCount >= 10;
119
-
120
- let keys = lifecycle.entries.map(x => x.groupByKeys.map(y => y.ourKey).concat(Object.keys(x.variables))).flat();
121
-
122
- function valueHash(key: string, value: unknown) {
123
- return getPathStr2(key, String(value));
124
- }
125
- let values = new Map<string, { key: string; value: unknown }>();
126
- for (let entry of firstInstance.entries) {
127
- for (let [key, value] of Object.entries(entry.datum)) {
128
- if (!keys.includes(key)) continue;
129
- if (key === matchedKey) continue;
130
- if (["__threadId", "__machineId"].includes(key)) continue;
131
- values.set(valueHash(key, value), { key, value });
132
- }
133
- }
134
-
135
- return <>
136
- <div className={css.hbox(4)}>
137
- {hitLimit && <span className={css.colorhsl(30, 80, 40).boldStyle}>(hit limit!)</span>}
138
- </div>
139
- {Array.from(values.values()).map(({ key, value }, idx) => (
140
- <div key={idx} className={css.hbox(4)}>
141
- <span className={css.colorhsl(200, 50, 60).boldStyle}>{key}:</span>
142
- <span className={css.colorhsl(0, 0, 50)}>{niceStringify(value)}</span>
143
- </div>
144
- ))}
145
- </>;
146
- })()}
147
- </>;
56
+ let additionalSearch = formatSearchString({ [matchedKey]: value });
57
+
58
+ return <ATag
59
+ key={lifecycle.id}
60
+ values={[
61
+ lifecycleIdURL.getOverride(lifecycle.id),
62
+ limitURL.getOverride(10),
63
+ additionalSearchURL.getOverride(additionalSearch),
64
+ phase2AdditionalSearchURL.getOverride("")
65
+ ]}
66
+ className={css.pad2(8, 4).hsl(180, 60, 90).hslhover(180, 60, 85).colorhsl(180, 80, 30).textDecoration("none").borderRadius(4).display("inline-block")}
67
+ >
68
+ {lifecycle.title}
69
+ </ATag>;
148
70
  })}
149
71
  </>;
150
72
  }
@@ -42,6 +42,8 @@ export function getLifecycleMatchesForDatum(
42
42
  entry,
43
43
  datum: extractedDatum,
44
44
  });
45
+ // For now, this is fine. Sometimes we actually have overlapping entries, which is bad, but doesn't really matter, And if we don't break here, we'll get multiple matches. And all of our entries have the same group by keys and variables anyway, so if we match one, it's the same as matching all of them.
46
+ break;
45
47
  }
46
48
  }
47
49
 
@@ -89,9 +89,10 @@ export function createLifeCycleSearch(
89
89
  let searchLifeCycle = async (params: {
90
90
  lifeCycleId: string;
91
91
  additionalSearch: string;
92
+ phase2AdditionalSearch: string;
92
93
  limit: number;
93
94
  }) => {
94
- let { lifeCycleId, additionalSearch, limit } = params;
95
+ let { lifeCycleId, additionalSearch, phase2AdditionalSearch, limit } = params;
95
96
  let lifeCycles = await context.controller.getLifeCycles.promise();
96
97
  if (!lifeCycles) return;
97
98
 
@@ -128,6 +129,7 @@ export function createLifeCycleSearch(
128
129
  lifeCycle,
129
130
  phase1Results,
130
131
  currentSequenceNumber,
132
+ phase2AdditionalSearch,
131
133
  limit,
132
134
  });
133
135
 
@@ -164,10 +166,13 @@ export function createLifeCycleSearch(
164
166
  resetPhase1State();
165
167
  });
166
168
 
167
- let searchQuery = startEntries.map(entry => entry.matchPattern).join("|");
168
- if (additionalSearch) {
169
- searchQuery = searchQuery + "&" + additionalSearch;
170
- }
169
+ let searchQuery = startEntries.map(entry => {
170
+ let pattern = entry.matchPattern;
171
+ if (additionalSearch) {
172
+ pattern = pattern + "&" + additionalSearch;
173
+ }
174
+ return pattern;
175
+ }).join("|");
171
176
  let searchBuffer = Buffer.from(searchQuery, "utf8");
172
177
 
173
178
  let sourceTypes = new Set<string>();
@@ -298,9 +303,10 @@ export function createLifeCycleSearch(
298
303
  lifeCycle: LifeCycle;
299
304
  phase1Results: LogDatum[];
300
305
  currentSequenceNumber: number;
306
+ phase2AdditionalSearch: string;
301
307
  limit: number;
302
308
  }): Promise<{ results: LogDatum[]; hitLimit: boolean }> => {
303
- let { lifeCycle, phase1Results, currentSequenceNumber, limit } = params;
309
+ let { lifeCycle, phase1Results, currentSequenceNumber, phase2AdditionalSearch, limit } = params;
304
310
 
305
311
  if (phase1Results.length === 0) {
306
312
  return { results: [], hitLimit: false };
@@ -411,7 +417,12 @@ export function createLifeCycleSearch(
411
417
  sourceType === "warning" && loggers.warnLogs ||
412
418
  loggers.errorLogs;
413
419
 
414
- let searchQuery = Array.from(queries).join("|");
420
+ let searchQuery = Array.from(queries).map(query => {
421
+ if (phase2AdditionalSearch) {
422
+ return query + "&" + phase2AdditionalSearch;
423
+ }
424
+ return query;
425
+ }).join("|");
415
426
  let searchBuffer = Buffer.from(searchQuery, "utf8");
416
427
 
417
428
  let results: LogDatum[] = [];
@@ -15,6 +15,9 @@ export type LifeCycle = {
15
15
  entries: LifeCycleEntry[];
16
16
  };
17
17
 
18
+ // TODO: Actually, maybe we should make it so the group by keys and the variables are just in the life cycle, as we keep wanting them to be identical for all the entries. It would be a pretty simple refactor. Although we would have to retrofit the old data, But then again, we could just have the AI create a script for that...
19
+ // IF we do any changes, we SHOULD refactor it so we have the same groupByKeys and variables
20
+ // - Aliases for variables is very useful, and for group by keys. However, start key is not needed. We're never going to use that. We'll just change the logging code to log the key that we want.
18
21
  export type LifeCycleEntry = {
19
22
  description?: string;
20
23
 
@@ -13,7 +13,6 @@ import { t } from "../../src/2-proxy/schema2";
13
13
  import { SocketRegistered } from "socket-function/SocketFunctionTypes";
14
14
  import { nextId } from "socket-function/src/misc";
15
15
  import { IdentityController_getSecureIP, IdentityController_getMachineId } from "../-c-identity/IdentityController";
16
- import { LOCAL_DOMAIN } from "../0-path-value-core/PathController";
17
16
  import { getNextTime } from "../0-path-value-core/pathValueCore";
18
17
  import { encodeArgs } from "../3-path-functions/PathFunctionHelpers";
19
18
  import { CallSpec, getCurrentCall, getCurrentCallAllowUndefined, overrideCurrentCall } from "../3-path-functions/PathFunctionRunner";
@@ -28,6 +27,7 @@ import { addComponentButton } from "../5-diagnostics/qreactDebug";
28
27
  import { closeAllModals } from "../5-diagnostics/Modal";
29
28
  import { delay } from "socket-function/src/batching";
30
29
  import { currentViewParam, selectedServiceIdParam } from "../deployManager/urlParams";
30
+ import { FunctionCallInfo } from "./FunctionCallInfo";
31
31
 
32
32
  export const managementPageURL = new URLParam("managementpage", "");
33
33
  export const showingManagementURL = new URLParam("showingmanagement", false);
@@ -87,7 +87,7 @@ export async function registerManagementPages2(config: {
87
87
  getModule: () => import("./NodeViewer"),
88
88
  });
89
89
  inputPages.push({
90
- title: "Logs3",
90
+ title: "LOG VIEWER",
91
91
  componentName: "LogViewer3",
92
92
  getModule: () => import("./logs/IndexedLogs/LogViewer3"),
93
93
  });
@@ -142,6 +142,12 @@ export async function registerManagementPages2(config: {
142
142
  componentName: "RequireAuditPage",
143
143
  getModule: () => import("./misc-pages/RequireAuditPage"),
144
144
  });
145
+ inputPages.push({
146
+ title: "Sync Test",
147
+ componentName: "SyncTestPage",
148
+ controllerName: "SyncTestController",
149
+ getModule: () => import("./SyncTestPage"),
150
+ });
145
151
  inputPages.push(...config.pages);
146
152
 
147
153
  // NOTE: We don't store the UI in the database (here, or anywhere else, at least
@@ -343,7 +349,7 @@ class ManagementRoot extends qreact.Component {
343
349
  <ATag values={[
344
350
  managementPageURL.getOverride("MachinesPage"),
345
351
  currentViewParam.getOverride("services"),
346
- ]}>Deploy Machines</ATag>
352
+ ]}>Deploy Framework</ATag>
347
353
  <ATag values={[
348
354
  managementPageURL.getOverride("MachinesPage"),
349
355
  currentViewParam.getOverride("deploy"),
@@ -351,6 +357,7 @@ class ManagementRoot extends qreact.Component {
351
357
  {pages.map(page =>
352
358
  <ATag values={[{ param: managementPageURL, value: page.componentName }]}>{page.title}</ATag>
353
359
  )}
360
+ <FunctionCallInfo />
354
361
  </div>
355
362
  {currentPage &&
356
363
  <div
@@ -1,28 +1,24 @@
1
1
  import { DecodedValuePath, PathValueArchives, pathValueArchives } from "../../0-path-value-core/pathValueArchives";
2
2
  import { SocketFunction } from "socket-function/SocketFunction";
3
- import { AuthorityPath } from "../../0-path-value-core/NodePathAuthorities";
4
- import preact from "preact"; import { qreact } from "../../4-dom/qreact";
3
+ import { qreact } from "../../4-dom/qreact";
5
4
  import { logErrors } from "../../errors";
6
5
  import { getBrowserUrlNode } from "../../-f-node-discovery/NodeDiscovery";
7
- import { getPathFromStr, getPathStr, getPathStr1, isValidPathStr, rootPathStr } from "../../path";
8
- import { getDomain } from "../../config";
6
+ import { rootPathStr } from "../../path";
9
7
  import { css } from "typesafecss";
10
- import { cacheArgsEqual, cacheJSONArgsEqual, cacheLimited, cacheShallowConfigArgEqual, lazy } from "socket-function/src/caching";
8
+ import { cacheArgsEqual, cacheLimited, cacheShallowConfigArgEqual, lazy } from "socket-function/src/caching";
11
9
  import { atomic, atomicObjectWrite, atomicObjectWriteNoFreeze, doAtomicWrites, proxyWatcher } from "../../2-proxy/PathValueProxyWatcher";
12
10
  import { Querysub } from "../../4-querysub/QuerysubController";
13
11
  import { formatDate, formatDateTime, formatNumber, formatPercent, formatTime } from "socket-function/src/formatting/format";
14
- import { getSeededRandom } from "../../misc/random";
15
12
  import { Histogram, XYValue } from "../../library-components/Histogram";
16
13
  import { Button } from "../../library-components/Button";
17
14
  import { createURLSync } from "../../library-components/URLParam";
18
- import { Input } from "../../library-components/Input";
19
- import { deepCloneJSON, nextId, sort, throttleFunction, timeInDay } from "socket-function/src/misc";
20
- import { isDefined, isEmpty } from "../../misc";
21
- import { PathValue, Time, authorityStorage, epochTime } from "../../0-path-value-core/pathValueCore";
15
+ import { sort, throttleFunction } from "socket-function/src/misc";
16
+ import { isEmpty } from "../../misc";
17
+ import { PathValue, epochTime } from "../../0-path-value-core/pathValueCore";
22
18
  import { pathValueSerializer } from "../../-h-path-value-serialize/PathValueSerializer";
23
- import { InputLabel, InputLabelURL } from "../../library-components/InputLabel";
24
- import { addEpsilons, getLowUint32, getShortNumber } from "../../bits";
25
- import { measureBlock, measureFnc, measureWrap } from "socket-function/src/profiling/measure";
19
+ import { InputLabel } from "../../library-components/InputLabel";
20
+ import { addEpsilons } from "../../bits";
21
+ import { measureWrap } from "socket-function/src/profiling/measure";
26
22
  import { ArchiveViewerTable } from "./ArchiveViewerTable";
27
23
  import { liveWatchPaths, skipStringsURL, skipValuesURL, startTime, endTime, addFileCount, archiveFileValue, filtersURL, viewMode, archiveViewerDiskCache } from "./archiveViewerShared";
28
24
  import { ArchiveViewerFilterUI, filterDatums } from "./archiveViewerFilter";
@@ -33,6 +29,8 @@ import { t } from "../../4-querysub/Querysub";
33
29
  import { Anchor } from "../../library-components/ATag";
34
30
  import { assertIsManagementUser } from "../../diagnostics/managementPages";
35
31
  import { ButtonSelector } from "../../library-components/ButtonSelector";
32
+ import { AuthoritySpec } from "../../0-path-value-core/PathRouter";
33
+ import { getAllAuthoritySpec } from "../../0-path-value-core/PathRouterServerAuthoritySpec";
36
34
 
37
35
  // NOTE: We can load up to about 1M PathValues (although it really depends on their contents,
38
36
  // specifically how many unique paths there are). After that the browser will likely stop
@@ -103,11 +101,7 @@ export class ArchiveViewer extends qreact.Component {
103
101
  return;
104
102
  }
105
103
  try {
106
- let newValuePaths = await controller.getValuePaths({
107
- //pathPrefix: getPathStr1(getDomain()),
108
- // More paths is probably better, as it might catch corrupted or missing domain files?
109
- pathPrefix: rootPathStr,
110
- });
104
+ let newValuePaths = await controller.getValuePaths(getAllAuthoritySpec());
111
105
  Querysub.localCommit(() => this.state.valuePaths = newValuePaths);
112
106
  } catch (e: any) {
113
107
  Querysub.localCommit(() => {
@@ -190,8 +184,7 @@ export class ArchiveViewer extends qreact.Component {
190
184
  this.state.parsedValues = atomicObjectWrite({});
191
185
  });
192
186
 
193
- // NOTE: I don't understand why we were grouping this. Doing a thousand calls is not that slow, there's not that much overhead. And we want to have small groups, that way, we get more progress.
194
- const MAX_FILES = 1;
187
+ const MAX_FILES = 50;
195
188
  const MAX_SIZE = 500 * 1024 * 1024;
196
189
  let bufferObjs: { path: string; buffer: Buffer }[] = [];
197
190
  let newParsedValues: { [path: string]: PathValue[] } = {};
@@ -249,14 +242,15 @@ export class ArchiveViewer extends qreact.Component {
249
242
  this.state.progress.filesParsed.max = remainingFiles.size + bufferObjs.length;
250
243
  });
251
244
 
252
- await Promise.all(fileGroups.map(async group => {
253
- //let time = Date.now();
245
+ for (let group of fileGroups) {
246
+ let time = Date.now();
247
+ console.log(`Downloading ${group.file.length} files, ${formatNumber(group.size)}B, at ${formatDateTime(time)}`);
254
248
  let newBuffers = await controller.getRawValueFiles(group.file, {
255
249
  // 1) Allows viewing old archives
256
250
  // 2) Allows viewing data that is being updated frequently more easily?
257
251
  includeRecycleBin: true
258
252
  });
259
- //console.log(`Downloaded ${formatNumber(newBuffers.map(x => x.length).reduce((a, b) => a + b, 0))}B in ${formatTime(Date.now() - time)}`);
253
+ console.log(`Downloaded ${formatNumber(newBuffers.map(x => x.length).reduce((a, b) => a + b, 0))}B in ${formatTime(Date.now() - time)}`);
260
254
  if (newBuffers.length !== group.file.length) {
261
255
  throw new Error(`Expected ${group.file.length} buffers, got ${newBuffers.length}`);
262
256
  }
@@ -269,7 +263,7 @@ export class ArchiveViewer extends qreact.Component {
269
263
  console.log(`Missing buffer for ${file}, skipping`);
270
264
  continue;
271
265
  }
272
- console.log(`Downloaded ${file}, ${formatNumber(buffer.length)}B`);
266
+ //console.log(`Downloaded ${file}, ${formatNumber(buffer.length)}B`);
273
267
  archiveViewerDiskCache().set(file, buffer);
274
268
  bufferObjs.push({ path: file, buffer });
275
269
  }
@@ -277,7 +271,7 @@ export class ArchiveViewer extends qreact.Component {
277
271
  this.state.progress.filesDownloaded.value += group.file.length;
278
272
  this.state.progress.bytesDownloaded += newBuffers.reduce((sum, buffer) => sum + (buffer?.buffer.byteLength || 0), 0);
279
273
  });
280
- }));
274
+ }
281
275
  Querysub.commitLocal(() => {
282
276
  this.state.progress.filesDownloaded.endTime = Date.now();
283
277
  });
@@ -720,7 +714,7 @@ class ProcessProgress extends qreact.Component<{
720
714
 
721
715
  const EMPTY_BUFFER = Buffer.alloc(0);
722
716
  class ArchiveViewerControllerBase {
723
- public async getValuePaths(authority: AuthorityPath): Promise<string[]> {
717
+ public async getValuePaths(authority: AuthoritySpec): Promise<string[]> {
724
718
  return pathValueArchives.getValuePaths(authority);
725
719
  }
726
720
  public async getValuePathSizes(files: string[]): Promise<number[]> {
@@ -17,7 +17,7 @@ import { pathValueSerializer } from "../../-h-path-value-serialize/PathValueSeri
17
17
  import { getSingleSizeEstimate } from "../../5-diagnostics/shared";
18
18
  import { ButtonSelector } from "../../library-components/ButtonSelector";
19
19
 
20
- const archiveTreeScale = createURLSync("archiveTreeScale", "size" as "count" | "equal" | "size");
20
+ const archiveTreeScale = createURLSync("archiveTreeScale", "count" as "count" | "equal" | "size");
21
21
  const archiveTreeSelected = createURLSync("archiveTreeSelected", "");
22
22
  const archiveMinPx = createURLSync("archiveMinPx", 50);
23
23
  const forceWildcard = createURLSync("forceWildcard", "");
@@ -82,9 +82,6 @@ export class ArchiveViewerTree extends qreact.Component<{
82
82
  if (scaleType === "size") {
83
83
  total = node.totalSizeEstimate;
84
84
  }
85
- if (node.values.length === 1 && total > 1000 * 1000 * 1000) {
86
- debugger;
87
- }
88
85
  countInfo += `(${formatNumber(node.values.length)} /// ${sumFormatter(total)}) `;
89
86
  } else {
90
87
  countInfo += `(${formatNumber(node.values.length)}) `;
@@ -423,6 +420,11 @@ export class ArchiveViewerTree extends qreact.Component<{
423
420
  <ATag values={[{ param: viewMode, value: "table" }, { param: filtersURL, value: exactFilter }]}>
424
421
  View Exact Path
425
422
  </ATag>
423
+ <Button onClick={() => {
424
+ void navigator.clipboard.writeText(archiveTreeSelected.value);
425
+ }}>
426
+ Copy To Clipboard
427
+ </Button>
426
428
  {lastFocused && !lite && <ATag values={[{ param: viewMode, value: "inspect" }, getSetArchiveSelected(lastFocused)]}>
427
429
  Inspect Latest
428
430
  </ATag>}
@@ -1,6 +1,6 @@
1
1
  import { ExternalRenderClass, qreact } from "../../4-dom/qreact";
2
2
  import { throttleRerender } from "../../functional/throttleRerender";
3
- import { pathWatcher } from "../../0-path-value-core/pathValueCore";
3
+ import { pathWatcher } from "../../0-path-value-core/PathWatcher";
4
4
  import { css } from "../../4-dom/css";
5
5
  import { TimeRangeView } from "./TimeRangeView";
6
6
  import { URLParam } from "../../library-components/URLParam";
@@ -13,7 +13,7 @@ import { managementPageURL, showingManagementURL } from "../../diagnostics/manag
13
13
  import { watchShowLocalPaths, watchValueType } from "./LocalWatchViewer";
14
14
  import { filtersURL, viewMode } from "./archiveViewerShared";
15
15
  import { proxyWatcher } from "../../2-proxy/PathValueProxyWatcher";
16
- import { LOCAL_DOMAIN_PATH } from "../../0-path-value-core/NodePathAuthorities";
16
+ import { LOCAL_DOMAIN_PATH } from "../../0-path-value-core/PathRouter";
17
17
 
18
18
  const historySecondsURL = new URLParam("historySeconds", 10);
19
19
 
@@ -1,12 +1,12 @@
1
- import { PathValue, ReadLock, authorityStorage, epochTime, pathValueArchives, pathWatcher } from "../../0-path-value-core/pathValueCore";
2
- import { PathValueProxyWatcher, doAtomicWrites, proxyWatcher } from "../../2-proxy/PathValueProxyWatcher";
1
+ import { PathValue, ReadLock, authorityStorage, epochTime } from "../../0-path-value-core/pathValueCore";
2
+ import { pathWatcher } from "../../0-path-value-core/PathWatcher";
3
+ import { PathValueProxyWatcher, proxyWatcher } from "../../2-proxy/PathValueProxyWatcher";
3
4
  import { ExternalRenderClass, qreact } from "../../4-dom/qreact";
4
- import { Querysub } from "../../4-querysub/Querysub";
5
5
  import { Table } from "../../5-diagnostics/Table";
6
6
  import { appendToPathStr, getPathFromStr, getPathStr } from "../../path";
7
7
  import { css } from "typesafecss";
8
8
  import { filtersURL, formatPath, viewMode } from "./archiveViewerShared";
9
- import { nextId, sort } from "socket-function/src/misc";
9
+ import { nextId } from "socket-function/src/misc";
10
10
  import { ArchiveViewerFilterUI, filterDatums } from "./archiveViewerFilter";
11
11
  import { ArchiveViewerTree } from "./ArchiveViewerTree";
12
12
  import { Button } from "../../library-components/Button";
@@ -15,12 +15,10 @@ import { Anchor } from "../../library-components/ATag";
15
15
  import { ButtonSelector } from "../../library-components/ButtonSelector";
16
16
  import { throttleRerender } from "../../functional/throttleRerender";
17
17
  import { InputLabel, InputLabelURL } from "../../library-components/InputLabel";
18
- import { formatPercent, formatTime } from "socket-function/src/formatting/format";
19
- import { LOCAL_DOMAIN_PATH } from "../../0-path-value-core/NodePathAuthorities";
20
- import { TimeRange, TimeRangeView, getTimeRangeStats, getUniqueTimeByPath, normalizeTimeRanges } from "./TimeRangeView";
21
- import { ComponentSyncStatsBase } from "./ComponentSyncStats";
18
+ import { formatTime } from "socket-function/src/formatting/format";
19
+ import { TimeRange, TimeRangeView, getUniqueTimeByPath, normalizeTimeRanges } from "./TimeRangeView";
22
20
  import { addEpsilons } from "../../bits";
23
- import { pathValueSerializer } from "../../-h-path-value-serialize/PathValueSerializer";
21
+ import { LOCAL_DOMAIN_PATH } from "../../0-path-value-core/PathRouter";
24
22
 
25
23
  export const watchValueType = new URLParam("valueType", "watch" as "watch" | "component");
26
24
  export const watchShowLocalPaths = new URLParam("showLocalPaths", false);
@@ -3,11 +3,10 @@ module.allowclient = true;
3
3
  import { ArchiveSnapshotOverview, ArchiveSnapshotRead, getSnapshot, getSnapshotList, loadSnapshot, saveSnapshot } from "../../0-path-value-core/archiveLocks/archiveSnapshots";
4
4
  import { qreact } from "../../4-dom/qreact";
5
5
  import { SocketFunction } from "socket-function/SocketFunction";
6
- import { ValueAuditController } from "../../5-diagnostics/memoryValueAudit";
7
6
  import { getBrowserUrlNode } from "../../-f-node-discovery/NodeDiscovery";
8
7
  import { Table } from "../../5-diagnostics/Table";
9
8
  import { css } from "typesafecss";
10
- import { t } from "../../4-querysub/Querysub";
9
+ import { Querysub, t } from "../../4-querysub/Querysub";
11
10
  import { ATag, Anchor } from "../../library-components/ATag";
12
11
  import { URLParam } from "../../library-components/URLParam";
13
12
  import { formatDateTime, formatNumber } from "socket-function/src/formatting/format";
@@ -29,7 +28,8 @@ const selectedSnapshot = new URLParam("selectedSnapshot", "");
29
28
  export class SnapshotViewer extends qreact.Component {
30
29
  state = t.state({
31
30
  // file => true
32
- expanded: t.lookup(t.boolean)
31
+ expanded: t.lookup(t.boolean),
32
+ saving: t.boolean(false),
33
33
  });
34
34
  render() {
35
35
  let controller = SnapshotViewerSynced(getBrowserUrlNode());
@@ -39,7 +39,7 @@ export class SnapshotViewer extends qreact.Component {
39
39
  if (!file || typeof file !== "string") {
40
40
  return (
41
41
  <div class={css.pad2(10).vbox(10)}>
42
- <h1>Snapshots are taken after every change (with the new data).</h1>
42
+ <h1>Snapshots are taken before every change. "zombie" means the file exists, but it has no confirm (so it will be ignored).</h1>
43
43
  <Table
44
44
  rows={snapshotList}
45
45
  columns={{
@@ -56,10 +56,16 @@ export class SnapshotViewer extends qreact.Component {
56
56
  </ATag>
57
57
  {context?.row?.file === "live" &&
58
58
  <Button onClick={async () => {
59
- await SnapshotViewerController.nodes[getBrowserUrlNode()].saveLiveSnapshot();
60
- console.log("Saved live snapshot.");
59
+ this.state.saving = true;
60
+ try {
61
+ await SnapshotViewerSynced(getBrowserUrlNode()).saveLiveSnapshot.promise();
62
+ } finally {
63
+ Querysub.localCommit(() => {
64
+ this.state.saving = false;
65
+ });
66
+ }
61
67
  }}>
62
- Save
68
+ {this.state.saving ? "Saving..." : "Save"}
63
69
  </Button>
64
70
  }
65
71
  </div>;
@@ -155,9 +161,9 @@ async function seriousConfirm(config: {
155
161
  </pre>}
156
162
  onChangeValue={value => {
157
163
  if (value === "confirm") {
158
- modal.close();
159
164
  resolve(true);
160
165
  }
166
+ modal.close();
161
167
  }}
162
168
  />
163
169
  </FullscreenModal>
@@ -175,9 +181,6 @@ class SnapshotViewerControllerBase {
175
181
  }): Promise<void> {
176
182
  await loadSnapshot({
177
183
  overview: config.overview,
178
- async audit(nodeId) {
179
- await ValueAuditController.nodes[nodeId].diskAuditNow();
180
- },
181
184
  });
182
185
  }
183
186
 
@@ -207,4 +210,12 @@ export const SnapshotViewerController = SocketFunction.register(
207
210
  noAutoExpose: true,
208
211
  }
209
212
  );
210
- const SnapshotViewerSynced = getSyncedController(SnapshotViewerController);
213
+ const SnapshotViewerSynced = getSyncedController(SnapshotViewerController, {
214
+ reads: {
215
+ getSnapshotList: ["snapshots"],
216
+ getSnapshot: ["snapshots"],
217
+ },
218
+ writes: {
219
+ saveLiveSnapshot: ["snapshots"],
220
+ }
221
+ });
@@ -12,7 +12,7 @@ import { css } from "typesafecss";
12
12
 
13
13
  export const startTime = createURLSync("archiveStartTime", 0);
14
14
  export const endTime = createURLSync("archiveEndTime", +new Date("2100/01/01 00:00:00"));
15
- export const archiveFileValue = createURLSync("archiveFileValue", "files" as "files" | "datums" | "bytes");
15
+ export const archiveFileValue = createURLSync("archiveFileValue", "datums" as "files" | "datums" | "bytes");
16
16
  export const filtersURL = createURLSync("archivefilters2", {} as { [key: string]: FilterPart });
17
17
 
18
18
  export const liveWatchPaths = createURLSync("liveWatchPaths", {} as { [path: string]: true });