querysub 0.719.0 → 0.720.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json
CHANGED
|
@@ -12,7 +12,7 @@ import { MaybePromise } from "socket-function/src/types";
|
|
|
12
12
|
import yargs from "yargs";
|
|
13
13
|
import { SyncWatcher } from "../2-proxy/PathValueProxyWatcher";
|
|
14
14
|
import { MAX_CHANGE_AGE } from "../0-path-value-core/pathValueCore";
|
|
15
|
-
import { isNode, isNodeTrue, nextId, timeInSecond } from "socket-function/src/misc";
|
|
15
|
+
import { isNode, isNodeTrue, nextId, timeInMinute, timeInSecond } from "socket-function/src/misc";
|
|
16
16
|
import { getPathStr2, getPathStr3 } from "../path";
|
|
17
17
|
import { consistentHash } from "../misc/hash";
|
|
18
18
|
import { setExternalHotReloading } from "socket-function/hot/HotReloadController";
|
|
@@ -22,10 +22,15 @@ import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
|
|
|
22
22
|
import { getControllerNodeId, getControllerNodeIdList } from "../-g-core-values/NodeCapabilities";
|
|
23
23
|
import { sha256 } from "js-sha256";
|
|
24
24
|
import os from "os";
|
|
25
|
-
import { formatTime } from "socket-function/src/formatting/format";
|
|
25
|
+
import { formatDateTime, formatTime } from "socket-function/src/formatting/format";
|
|
26
26
|
import path from "path";
|
|
27
27
|
import pathMod from "path";
|
|
28
28
|
|
|
29
|
+
const MODULE_LOAD_ERROR_RETRY_DELAY = timeInMinute * 5;
|
|
30
|
+
const YARN_INSTALL_INITIAL_RETRY_DELAY = timeInSecond * 10;
|
|
31
|
+
const YARN_INSTALL_MAX_RETRY_DELAY = timeInMinute;
|
|
32
|
+
const YARN_INSTALL_MAX_ATTEMPTS = 10;
|
|
33
|
+
|
|
29
34
|
export type LoadFunctionSpec = {
|
|
30
35
|
gitURL: string;
|
|
31
36
|
gitRef: string;
|
|
@@ -87,6 +92,7 @@ export function getModuleFromSpec(spec: FunctionSpec): MaybePromise<NodeJS.Modul
|
|
|
87
92
|
let moduleCache = new Map<string, {
|
|
88
93
|
result: MaybePromise<NodeJS.Module>;
|
|
89
94
|
error: Error | undefined;
|
|
95
|
+
errorTime: number;
|
|
90
96
|
}>();
|
|
91
97
|
function getSpecKey(spec: LoadFunctionSpec) {
|
|
92
98
|
// Only include these specific keys AND order them, so the hash hits no matter how this function is called
|
|
@@ -104,18 +110,23 @@ export function getModuleFromConfig(spec: LoadFunctionSpec): MaybePromise<NodeJS
|
|
|
104
110
|
let value = moduleCache.get(key);
|
|
105
111
|
if (!value) {
|
|
106
112
|
let promise = getModuleFromSpecBase(spec);
|
|
107
|
-
value = { result: promise, error: undefined };
|
|
113
|
+
value = { result: promise, error: undefined, errorTime: 0 };
|
|
108
114
|
moduleCache.set(key, value);
|
|
109
115
|
promise.then(
|
|
110
116
|
result => {
|
|
111
|
-
moduleCache.set(key, { result, error: undefined });
|
|
117
|
+
moduleCache.set(key, { result, error: undefined, errorTime: 0 });
|
|
112
118
|
},
|
|
113
119
|
error => {
|
|
114
|
-
moduleCache.set(key, { result:
|
|
120
|
+
moduleCache.set(key, { result: promise, error, errorTime: Date.now() });
|
|
115
121
|
}
|
|
116
122
|
);
|
|
117
123
|
}
|
|
118
|
-
if (value.error)
|
|
124
|
+
if (value.error) {
|
|
125
|
+
if (Date.now() - value.errorTime < MODULE_LOAD_ERROR_RETRY_DELAY) throw value.error;
|
|
126
|
+
console.log(red(`Retrying module load for ${key}, the previous attempt failed at ${formatDateTime(value.errorTime)}, more than ${formatTime(MODULE_LOAD_ERROR_RETRY_DELAY)} ago, with:\n${value.error.stack}`));
|
|
127
|
+
moduleCache.delete(key);
|
|
128
|
+
return getModuleFromConfig(spec);
|
|
129
|
+
}
|
|
119
130
|
return value.result;
|
|
120
131
|
}
|
|
121
132
|
|
|
@@ -466,17 +477,33 @@ const hotreloadIfChanged = batchFunction({ delay: 100, name: "hotreloadIfChanged
|
|
|
466
477
|
// A corrupt entry in the shared global yarn cache (e.g. "Extracting tar content ... the file appears to be
|
|
467
478
|
// corrupt") fails every install identically, because re-cloning the repo folder never touches the global
|
|
468
479
|
// cache. Detect that, purge the cache with `yarn cache clean`, and retry once so the loader heals itself.
|
|
480
|
+
// Retrying with delays could make this function very slow. However, the alternative is just all the function calls failing, which is the worst case scenario.
|
|
469
481
|
async function yarnInstallWithHealing(repoPath: string): Promise<void> {
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
482
|
+
let cleanedCache = false;
|
|
483
|
+
let failures = 0;
|
|
484
|
+
let retryDelay = YARN_INSTALL_INITIAL_RETRY_DELAY;
|
|
485
|
+
while (true) {
|
|
486
|
+
try {
|
|
487
|
+
await executeCommand("yarn", ["install", "--mutex", "network"], { cwd: repoPath });
|
|
488
|
+
if (failures) {
|
|
489
|
+
console.log(blue(`yarn install in ${repoPath} succeeded after ${failures} failures`));
|
|
490
|
+
}
|
|
491
|
+
return;
|
|
492
|
+
} catch (e) {
|
|
493
|
+
failures++;
|
|
494
|
+
let errorText = String((e as Error)?.stack || e);
|
|
495
|
+
if (failures >= YARN_INSTALL_MAX_ATTEMPTS) throw e;
|
|
496
|
+
if (!cleanedCache && /appears to be corrupt|Extracting tar content/.test(errorText)) {
|
|
497
|
+
cleanedCache = true;
|
|
498
|
+
console.log(red(`yarn install failed with a corrupt yarn cache, cleaning cache and retrying:\n${errorText}`));
|
|
499
|
+
await executeCommand("yarn", ["cache", "clean"]);
|
|
500
|
+
continue;
|
|
501
|
+
}
|
|
502
|
+
console.log(red(`yarn install in ${repoPath} failed (failure ${failures}/${YARN_INSTALL_MAX_ATTEMPTS}), retrying in ${formatTime(retryDelay)}:\n${errorText}`));
|
|
503
|
+
await delay(retryDelay);
|
|
504
|
+
retryDelay = Math.min(retryDelay * 2, YARN_INSTALL_MAX_RETRY_DELAY);
|
|
505
|
+
}
|
|
478
506
|
}
|
|
479
|
-
await executeCommand("yarn", ["install", "--mutex", "network"], { cwd: repoPath });
|
|
480
507
|
}
|
|
481
508
|
|
|
482
509
|
async function which(command: string): Promise<string> {
|
|
@@ -8,6 +8,7 @@ import { sort, timeInSecond, nextId } from "socket-function/src/misc";
|
|
|
8
8
|
import { MachineController, watchProcessOutput, stopWatchingProcessOutput } from "../machineController";
|
|
9
9
|
import type { ProcessRecord } from "../processLogs";
|
|
10
10
|
import { Button } from "../../library-components/Button";
|
|
11
|
+
import { getMachineInfoLive, getMachineNameLive } from "../machineSchema";
|
|
11
12
|
import { parseAnsiColors, rgbToHsl } from "../../diagnostics/logs/ansiFormat";
|
|
12
13
|
|
|
13
14
|
module.hotreload = true;
|
|
@@ -168,9 +169,11 @@ class MachineProcesses extends qreact.Component<{ machineId: string; applyNodeId
|
|
|
168
169
|
let historical = shown.filter(x => x.deadTime !== undefined);
|
|
169
170
|
sort(running, x => -x.startTime);
|
|
170
171
|
sort(historical, x => -(x.deadTime || 0));
|
|
172
|
+
let machineName = getMachineNameLive(this.props.machineId);
|
|
171
173
|
return <div className={css.vbox(8).fillWidth}>
|
|
172
174
|
<div className={css.hbox(10).wrap.alignItems("center")}>
|
|
173
|
-
<div className={css.boldStyle}>{
|
|
175
|
+
<div className={css.boldStyle + (!machineName && css.colorhsl(0, 0, 45))}>{machineName || "(unnamed machine)"}</div>
|
|
176
|
+
<div>{this.props.machineId} ({getMachineInfoLive(this.props.machineId)?.info["getExternalIP"]})</div>
|
|
174
177
|
<div className={css.colorhsl(0, 0, 45)}>{running.length} running</div>
|
|
175
178
|
{historical.length > 0 && <Button flavor="tiny" onClick={() => this.state.showHistorical = !this.state.showHistorical}>
|
|
176
179
|
{this.state.showHistorical && `Hide ${historical.length} historical` || `Show ${historical.length} historical`}
|