probe-research 0.42.0 → 0.43.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -34,6 +34,7 @@ const path = require("node:path");
34
34
  const agentRun = require("../lib/agent-run");
35
35
  const approval = require("../lib/approval-watch");
36
36
  const download = require("../lib/background-download");
37
+ const pluginPrefetch = require("../lib/plugin-prefetch");
37
38
  const setupCheck = require("../lib/setup-check");
38
39
  const signIn = require("../lib/sign-in");
39
40
  /** Probe's own upkeep (`PROBE_UPKEEP`), as `make sync-probe-upkeep` writes it from the CLI's flags. */
@@ -625,9 +626,10 @@ function readGlibc() {
625
626
  const REPAIR_NEEDS_NETWORK = "probe repair needs the network to fetch a working CLI";
626
627
 
627
628
  async function main() {
628
- const startedMs = Date.now();
629
+ const startedMs = agentRun.getClockMs(); // the budget's clock: pauses in sleep (plan I-F1)
629
630
  const args = process.argv.slice(2);
630
631
  if (args[0] === download.DOWNLOAD_VERB) return runBackgroundDownload();
632
+ if (args[0] === pluginPrefetch.PREFETCH_VERB) process.exit(await pluginPrefetch.runPrefetch());
631
633
  if (args[0] === EVENT_VERB) return sendBackgroundEvent(args.slice(1));
632
634
  if (args[0] === LauncherVerb.SIGN_IN) return runSignIn(args.slice(1), startedMs);
633
635
  // `npx probe-research` with no arguments runs the wizard — that is the entire
@@ -758,6 +760,8 @@ const APPROVAL_READ_RESERVE_MS = signIn.APPROVAL_TIMEOUT_MS + 1000;
758
760
  * so the agent runs it again instead (#4661 review: 15 s for the start and the exchange, as before).
759
761
  */
760
762
  const APPROVED_RESERVE_MS = 30 * 1000 + APPROVAL_READ_RESERVE_MS;
763
+ /** The first download's extras: the CLI's (`all`) and the daemon's AI libraries (plan I-F2). */
764
+ const FIRST_DOWNLOAD_EXTRAS = "all,daemon";
761
765
  /** How often a running download is looked at again. */
762
766
  const DOWNLOAD_POLL_MS = 1000;
763
767
  /** The uninstall barrier (`probe.shared.updates.uninstall_barrier.path`). */
@@ -868,13 +872,19 @@ async function runAgentCall(cmd, cmdArgs, startedMs, step, watch = null) {
868
872
  process.exit(ran.code);
869
873
  }
870
874
 
871
- /** The newest release's `probe` arguments for uv, pinned when the manifest names it. */
872
- async function getLatestSpec() {
875
+ /**
876
+ * The newest release's `probe` arguments for uv, pinned when the manifest names it. With `daemon` (the first
877
+ * download's): the daemon's AI libraries too (+53 MB, measured 11-15), fetched behind the browser approval, so the
878
+ * install's who-records step never builds them (~70 s) mid-install (plan I-F2). The CLI's install from this copy
879
+ * keeps the extra (`updater._dist`).
880
+ */
881
+ async function getLatestSpec({ daemon = false } = {}) {
873
882
  const latest = await latestVersion();
874
883
  const missingBuild = getMissingBuild(process.platform, process.arch,
875
884
  process.platform === "linux" ? readGlibc() : null);
876
885
  const pinned = !missingBuild && latest && atLeast(latest, AGENT_MIN_CLI) ? latest : null;
877
- return { pinned, spec: pinned ? `${DIST}[all]==${pinned}` : `${DIST}[all]>=${AGENT_MIN_CLI}`, missingBuild };
886
+ const dist = `${DIST}[${daemon ? FIRST_DOWNLOAD_EXTRAS : "all"}]`;
887
+ return { pinned, spec: pinned ? `${dist}==${pinned}` : `${dist}>=${AGENT_MIN_CLI}`, missingBuild };
878
888
  }
879
889
 
880
890
  /** A uv this launcher runs: the one on PATH when it is new enough, else the pinned one installed. */
@@ -895,7 +905,7 @@ function startBackgroundDownload() {
895
905
  * own (an old uv that runs no `python` from a tool env): the second downloads too.
896
906
  */
897
907
  async function runBackgroundDownload() {
898
- const { pinned, spec, missingBuild } = await getLatestSpec();
908
+ const { pinned, spec, missingBuild } = await getLatestSpec({ daemon: true });
899
909
  const refresh = getRefreshFlags({ repair: false, missingBuild, pinned });
900
910
  const uv = getUv();
901
911
  process.exit(download.runDownload([
@@ -914,7 +924,7 @@ async function finishStoppedUninstall(deadlineMs) {
914
924
  const refresh = getRefreshFlags({ repair: true });
915
925
  spawnSync(uv, ["tool", "run", "--isolated", "--python", PYTHON_REQUEST, ...refresh, "--from", spec, "probe",
916
926
  "repair", "--json"],
917
- { stdio: ["ignore", 2, 2], timeout: Math.max(1000, deadlineMs - WAIT_RESERVE_MS - Date.now()) });
927
+ { stdio: ["ignore", 2, 2], timeout: Math.max(1000, deadlineMs - WAIT_RESERVE_MS - agentRun.getClockMs()) });
918
928
  if (hasBarrier()) {
919
929
  await refuseAgent(AgentStep.SIGN_IN, agentRun.Reason.UNINSTALL_RUNNING,
920
930
  "An uninstall of Probe stopped halfway and is being finished first: run the same command again.",
@@ -932,7 +942,7 @@ async function waitForRunningDownload(startedMs, step, watch = null) {
932
942
  await updateWatch(watch, step, isRunning ? record.stage || download.DownloadStage.DOWNLOADING : null);
933
943
  if (!isRunning) return record;
934
944
  // Not approved yet: the approval is read on to the budget's end, the download with it (fixes-1108b).
935
- if (Date.now() >= (watch && watch.isPending() ? getWatchEndMs(startedMs) : until)) {
945
+ if (agentRun.getClockMs() >= (watch && watch.isPending() ? getWatchEndMs(startedMs) : until)) {
936
946
  await refuseUnfinished(step, watch, record.stage || download.DownloadStage.DOWNLOADING);
937
947
  }
938
948
  await sleep(DOWNLOAD_POLL_MS);
@@ -950,7 +960,7 @@ function getWatchEndMs(startedMs) {
950
960
  * that comes meanwhile while Probe still installs is answered at once, as PR39b's.
951
961
  */
952
962
  async function watchUntilBudgetEnd(watch, startedMs, step, stage) {
953
- while (watch && watch.isPending() && Date.now() + DOWNLOAD_POLL_MS < getWatchEndMs(startedMs)) {
963
+ while (watch && watch.isPending() && agentRun.getClockMs() + DOWNLOAD_POLL_MS < getWatchEndMs(startedMs)) {
954
964
  await sleep(DOWNLOAD_POLL_MS);
955
965
  await updateWatch(watch, step, stage);
956
966
  }
@@ -1019,6 +1029,10 @@ async function runSignIn(rest, startedMs) {
1019
1029
  const started = await signIn.startSignIn({ noBrowser: rest.includes("--no-browser") });
1020
1030
  if (!started.ok) await refuseAgent(AgentStep.SIGN_IN, agentRun.Reason.INCOMPLETE, started.message, 1);
1021
1031
  const downloading = findProbe(AGENT_MIN_CLI) ? "not_needed" : startBackgroundDownload();
1032
+ // The agents' plugin downloads too, so the install's agents stage only links them (plan I-F3).
1033
+ try {
1034
+ pluginPrefetch.startPrefetch({ launcher: __filename });
1035
+ } catch {} // without it the agents download as before
1022
1036
  sendEventInBackground(process.env, SIGN_IN_STARTED_EVENT, { chat_agent: signIn.detectAgent(process.env) });
1023
1037
  if (!rest.includes("--json")) {
1024
1038
  agentRun.writeLine(`Approve this computer in the browser: ${started.verification_uri_complete}`);
@@ -1050,7 +1064,7 @@ async function findDownloadedProbe(done, startedMs, step, watch = null) {
1050
1064
  const until = startedMs + agentRun.AGENT_BUDGET_MS - WAIT_RESERVE_MS;
1051
1065
  for (;;) {
1052
1066
  const probe = findProbe(AGENT_MIN_CLI);
1053
- if (probe || !done || done.state !== download.DownloadState.DONE || Date.now() >= until) return probe;
1067
+ if (probe || !done || done.state !== download.DownloadState.DONE || agentRun.getClockMs() >= until) return probe;
1054
1068
  await updateWatch(watch, step, approval.InstallStage.STARTING);
1055
1069
  await sleep(DOWNLOAD_POLL_MS);
1056
1070
  }
@@ -1085,7 +1099,7 @@ async function waitForSignIn(json, startedMs) {
1085
1099
  * answer `waiting_for_approval` the moment it started. */
1086
1100
  async function sayInstalled(watch, startedMs) {
1087
1101
  await updateWatch(watch, AgentStep.SIGN_IN_WAIT, approval.InstallStage.DONE);
1088
- if (startedMs + agentRun.AGENT_BUDGET_MS - Date.now() >= APPROVED_RESERVE_MS) return;
1102
+ if (startedMs + agentRun.AGENT_BUDGET_MS - agentRun.getClockMs() >= APPROVED_RESERVE_MS) return;
1089
1103
  await watchUntilBudgetEnd(watch, startedMs, AgentStep.SIGN_IN_WAIT, approval.InstallStage.DONE);
1090
1104
  if (watch.isPending()) {
1091
1105
  await refuseAgent(AgentStep.SIGN_IN_WAIT, agentRun.Reason.WAITING_FOR_APPROVAL,
package/lib/agent-run.js CHANGED
@@ -1,9 +1,11 @@
1
1
  /**
2
2
  * An agent's command, kept inside its 110 s (plan D3, D67): the CLI stops itself between steps
3
- * at the deadline it is handed (`PROBE_AGENT_DEADLINE`); this is only the backstop for a step that
4
- * hangs. The child runs in its own process group so the whole group can be stopped: SIGTERM at the
5
- * deadline, SIGKILL 5 s later, then one JSON line saying to run it again. Never the interactive
6
- * wizard: a detached group loses the terminal.
3
+ * at the seconds left it is handed (`PROBE_AGENT_SECONDS_LEFT`); this is only the backstop for a step
4
+ * that hangs. The 110 s count on `getClockMs()`, which pauses while the computer sleeps (plan I-F1).
5
+ *
6
+ * The child runs in its own process group so the whole group can be stopped: SIGTERM at the deadline,
7
+ * SIGKILL 5 s later, then one JSON line saying to run it again. Never the interactive wizard: a detached
8
+ * group loses the terminal.
7
9
  *
8
10
  * One output rule for every agent command (D97): JSON lines on stdout, the last one the answer. The
9
11
  * launcher's own answer is the CLI's `done` event, `{v, event: "done", ok, reason, exit_code, ...}`, its
@@ -12,6 +14,7 @@
12
14
 
13
15
  const fs = require("node:fs");
14
16
  const { spawn } = require("node:child_process");
17
+ const { performance } = require("node:perf_hooks");
15
18
 
16
19
  /** The whole command's time, the download wait included. */
17
20
  const AGENT_BUDGET_MS = 110 * 1000;
@@ -19,7 +22,9 @@ const AGENT_BUDGET_MS = 110 * 1000;
19
22
  const TEST_BUDGET_ENV = "PROBE_LAUNCHER_TEST_BUDGET_S";
20
23
  /** After SIGTERM, this long before SIGKILL. */
21
24
  const KILL_GRACE_MS = 5 * 1000;
22
- /** The variable the CLI reads its deadline from (epoch seconds; `agent_install.budget.DEADLINE_ENV`). */
25
+ /** The seconds the CLI has left when it starts (`foundation.agent_time_left.SECONDS_LEFT_ENV`). */
26
+ const SECONDS_LEFT_ENV = "PROBE_AGENT_SECONDS_LEFT";
27
+ /** The same end on the wall clock (epoch seconds), for CLIs older than SECONDS_LEFT_ENV (`DEADLINE_ENV`). */
23
28
  const DEADLINE_ENV = "PROBE_AGENT_DEADLINE";
24
29
  /** The CLI's own reserve: it is told a deadline this much before the backstop's. */
25
30
  const CLI_MARGIN_MS = 3 * 1000;
@@ -79,16 +84,28 @@ function refuse(reason, message, code, extra = {}) {
79
84
  answer({ ok: false, reason, message, ...extra }, code);
80
85
  }
81
86
 
87
+ /** The budget's clock, in whole ms (spawnSync's `timeout` takes integers only): it pauses while the computer
88
+ * sleeps and never moves with the wall clock (a Mac asleep mid-install spent its 110 s on it: plan I-F1). */
89
+ function getClockMs() {
90
+ return Math.round(performance.now());
91
+ }
92
+
93
+ /** What a child is told it has left, `leftMs`: seconds left, and the same end on the wall clock for an old CLI. */
94
+ function toTimeLeftEnv(leftMs) {
95
+ const left = Math.max(0, leftMs);
96
+ return { [SECONDS_LEFT_ENV]: String(left / 1000), [DEADLINE_ENV]: String((Date.now() + left) / 1000) };
97
+ }
98
+
82
99
  /**
83
- * Run `cmd args` with the terminal's stdio, in its own process group, until `deadlineMs` (epoch
84
- * ms): the child's own exit status, or the backstop's `run_again` when it had to be stopped.
100
+ * Run `cmd args` with the terminal's stdio, in its own process group, until `deadlineMs` (on
101
+ * `getClockMs()`): the child's own exit status, or the backstop's `run_again` when it had to be stopped.
85
102
  */
86
103
  function runWithBackstop(cmd, args, { deadlineMs, env = process.env } = {}) {
87
104
  return new Promise((resolve) => {
88
105
  const child = spawn(cmd, args, {
89
106
  stdio: "inherit",
90
107
  detached: true,
91
- env: { ...env, [DEADLINE_ENV]: String((deadlineMs - CLI_MARGIN_MS) / 1000) },
108
+ env: { ...env, ...toTimeLeftEnv(deadlineMs - CLI_MARGIN_MS - getClockMs()) },
92
109
  });
93
110
  let stopped = false;
94
111
  const signalGroup = (signal) => {
@@ -105,7 +122,7 @@ function runWithBackstop(cmd, args, { deadlineMs, env = process.env } = {}) {
105
122
  stopped = true;
106
123
  signalGroup("SIGTERM");
107
124
  killTimer = setTimeout(() => signalGroup("SIGKILL"), KILL_GRACE_MS);
108
- }, Math.max(0, deadlineMs - Date.now()));
125
+ }, Math.max(0, deadlineMs - getClockMs()));
109
126
  child.on("error", (err) => {
110
127
  clearTimeout(stopTimer);
111
128
  for (const [signal, handler] of Object.entries(handlers)) process.off(signal, handler);
@@ -136,10 +153,13 @@ module.exports = {
136
153
  KILL_GRACE_MS,
137
154
  Reason,
138
155
  SCHEMA_VERSION,
156
+ SECONDS_LEFT_ENV,
139
157
  answer,
140
158
  answerRead,
159
+ getClockMs,
141
160
  refuse,
142
161
  runWithBackstop,
143
162
  toProgressLine,
163
+ toTimeLeftEnv,
144
164
  writeLine,
145
165
  };
@@ -15,8 +15,9 @@ const fs = require("node:fs");
15
15
  const path = require("node:path");
16
16
  const { spawn, spawnSync } = require("node:child_process");
17
17
 
18
- const { AGENT_BUDGET_MS, DEADLINE_ENV } = require("./agent-run");
18
+ const { AGENT_BUDGET_MS, getClockMs, toTimeLeftEnv } = require("./agent-run");
19
19
  const { getStateFolder } = require("./sign-in");
20
+ const { isAlive, readLockHolder, releaseLock, takeLock } = require("./file-lock");
20
21
 
21
22
  const RECORD_NAME = "download.json";
22
23
  const LOG_NAME = "download.log";
@@ -68,56 +69,6 @@ function saveDownload(record) {
68
69
  fs.renameSync(temp, file);
69
70
  }
70
71
 
71
- function isAlive(pid) {
72
- if (!Number.isInteger(pid) || pid <= 1) return false;
73
- try {
74
- process.kill(pid, 0);
75
- return true;
76
- } catch (err) {
77
- return err && err.code === "EPERM";
78
- }
79
- }
80
-
81
- /** Take the one download's lock; false while a live download holds it (a stale one is broken). */
82
- function takeLock() {
83
- const lock = getFile(LOCK_NAME);
84
- fs.mkdirSync(getStateFolder(), { recursive: true, mode: 0o700 });
85
- for (let tries = 0; tries < 2; tries++) {
86
- try {
87
- const fd = fs.openSync(lock, "wx", 0o600);
88
- fs.writeSync(fd, String(process.pid));
89
- fs.closeSync(fd);
90
- return true;
91
- } catch {
92
- let holder = NaN;
93
- let age = Infinity;
94
- try {
95
- holder = parseInt(fs.readFileSync(lock, "utf8"), 10);
96
- age = Date.now() - fs.statSync(lock).mtimeMs;
97
- } catch {}
98
- if (isAlive(holder) && age < DOWNLOAD_CAP_MS) return false;
99
- try {
100
- fs.unlinkSync(lock);
101
- } catch {}
102
- }
103
- }
104
- return false;
105
- }
106
-
107
- function readLockHolder() {
108
- try {
109
- return fs.readFileSync(getFile(LOCK_NAME), "utf8");
110
- } catch {
111
- return "";
112
- }
113
- }
114
-
115
- function releaseLock() {
116
- try {
117
- if (parseInt(fs.readFileSync(getFile(LOCK_NAME), "utf8"), 10) === process.pid) fs.unlinkSync(getFile(LOCK_NAME));
118
- } catch {}
119
- }
120
-
121
72
  /**
122
73
  * Start the download in its own session, unless one runs: `"started" | "running"`. `args` are what
123
74
  * the detached launcher runs (`downloadArgs`), `launcher` its own file.
@@ -145,17 +96,18 @@ function startDownload({ launcher, args, env = process.env }) {
145
96
  * An optional step that fails is passed over (the next one does its work too). Returns the exit status.
146
97
  */
147
98
  function runDownload(steps) {
148
- if (!takeLock()) {
99
+ if (!takeLock(getFile(LOCK_NAME), DOWNLOAD_CAP_MS)) {
149
100
  // Another download runs: the record names it (the starter wrote this process's pid there).
150
- const holder = parseInt(readLockHolder(), 10);
101
+ const holder = readLockHolder(getFile(LOCK_NAME));
151
102
  if (isAlive(holder)) saveDownload({ state: DownloadState.RUNNING, pid: holder, started_at: Date.now() / 1000 });
152
103
  return 0;
153
104
  }
154
- const started = Date.now();
155
- saveDownload({ state: DownloadState.RUNNING, pid: process.pid, started_at: started / 1000 });
105
+ const startedAt = Date.now() / 1000;
106
+ const started = getClockMs();
107
+ saveDownload({ state: DownloadState.RUNNING, pid: process.pid, started_at: startedAt });
156
108
  try {
157
109
  for (const { stage, command, isOptional = false } of steps) {
158
- saveDownload({ state: DownloadState.RUNNING, stage, pid: process.pid, started_at: started / 1000 });
110
+ saveDownload({ state: DownloadState.RUNNING, stage, pid: process.pid, started_at: startedAt });
159
111
  let { ran, logged } = runStep(command, started);
160
112
  if (isOptional) continue;
161
113
  // Once more with a fresh deadline when its service never answered: uv's own download may have used up
@@ -170,18 +122,18 @@ function runDownload(steps) {
170
122
  saveDownload({ state: DownloadState.DONE, pid: process.pid, finished_at: Date.now() / 1000 });
171
123
  return 0;
172
124
  } finally {
173
- releaseLock();
125
+ releaseLock(getFile(LOCK_NAME));
174
126
  }
175
127
  }
176
128
 
177
- /** One step (`[cmd, ...args]`) within what is left of DOWNLOAD_CAP_MS: `{ran}` (null: no time left) and
178
- * where its output starts in the log (`logged`). */
129
+ /** One step (`[cmd, ...args]`) within what is left of DOWNLOAD_CAP_MS (`started`: on `getClockMs()`, which
130
+ * pauses in sleep): `{ran}` (null: no time left) and where its output starts in the log (`logged`). */
179
131
  function runStep([cmd, ...args], started) {
180
- const left = DOWNLOAD_CAP_MS - (Date.now() - started);
132
+ const left = DOWNLOAD_CAP_MS - (getClockMs() - started);
181
133
  const logged = getLogSize();
182
- // As an agent's call: the CLI's `probe` waits for a first service still starting until this deadline
134
+ // As an agent's call: the CLI's `probe` waits for a first service still starting for this long
183
135
  // (a fresh Mac's outlasted its own 10 s, so the install that runs inside it never ran: PR29).
184
- const env = { ...process.env, [DEADLINE_ENV]: String((Date.now() + AGENT_BUDGET_MS) / 1000) };
136
+ const env = { ...process.env, ...toTimeLeftEnv(AGENT_BUDGET_MS) };
185
137
  const ran = left > 0 ? spawnSync(cmd, args, { stdio: "inherit", env, timeout: left, killSignal: "SIGKILL" }) : null;
186
138
  return { ran, logged };
187
139
  }
@@ -0,0 +1,60 @@
1
+ /**
2
+ * One owner at a time for a detached launcher job (the first download, the plugin prefetch): a lock file
3
+ * holding the owner's pid, broken when that process is gone or the lock is older than the job's cap.
4
+ */
5
+
6
+ const fs = require("node:fs");
7
+ const path = require("node:path");
8
+
9
+ function isAlive(pid) {
10
+ if (!Number.isInteger(pid) || pid <= 1) return false;
11
+ try {
12
+ process.kill(pid, 0);
13
+ return true;
14
+ } catch (err) {
15
+ return err && err.code === "EPERM";
16
+ }
17
+ }
18
+
19
+ /** Take `lock` for this process; false while a live owner younger than `maxAgeMs` holds it (a stale one is broken). */
20
+ function takeLock(lock, maxAgeMs) {
21
+ fs.mkdirSync(path.dirname(lock), { recursive: true, mode: 0o700 });
22
+ for (let tries = 0; tries < 2; tries++) {
23
+ try {
24
+ const fd = fs.openSync(lock, "wx", 0o600);
25
+ fs.writeSync(fd, String(process.pid));
26
+ fs.closeSync(fd);
27
+ return true;
28
+ } catch {
29
+ let holder = NaN;
30
+ let age = Infinity;
31
+ try {
32
+ holder = parseInt(fs.readFileSync(lock, "utf8"), 10);
33
+ age = Date.now() - fs.statSync(lock).mtimeMs;
34
+ } catch {}
35
+ if (isAlive(holder) && age < maxAgeMs) return false;
36
+ try {
37
+ fs.unlinkSync(lock);
38
+ } catch {}
39
+ }
40
+ }
41
+ return false;
42
+ }
43
+
44
+ /** The pid `lock` names, or NaN. */
45
+ function readLockHolder(lock) {
46
+ try {
47
+ return parseInt(fs.readFileSync(lock, "utf8"), 10);
48
+ } catch {
49
+ return NaN;
50
+ }
51
+ }
52
+
53
+ /** Remove `lock` when this process holds it. */
54
+ function releaseLock(lock) {
55
+ try {
56
+ if (readLockHolder(lock) === process.pid) fs.unlinkSync(lock);
57
+ } catch {}
58
+ }
59
+
60
+ module.exports = { isAlive, readLockHolder, releaseLock, takeLock };
@@ -0,0 +1,299 @@
1
+ /**
2
+ * The agents' plugin downloads, behind the sign-in (plan I-F3): Probe's plugin marketplace as a bare git
3
+ * mirror (Claude Code's and Codex's first clone reads it) and its main branch's tarball (Cursor's and Kimi
4
+ * Code's copies), fetched while the person is in the browser tab. The install's agents stage then links
5
+ * them instead of downloading (37-67 s of a first install in the field, 2026-11-14).
6
+ *
7
+ * Only Probe's own folder is written: no agent's config, so a refused approval leaves the agents as they
8
+ * were. Its own process, in its own session, one at a time (`marketplaces.lock`), ended by itself at
9
+ * PREFETCH_CAP_MS; its record (`marketplaces.json`, pinned by `tests/fixtures/marketplace_prefetch.json`)
10
+ * is what the CLI reads (`shared/harness/marketplace_prefetch`). Any part that fails is left to the agent.
11
+ */
12
+
13
+ const fs = require("node:fs");
14
+ const http = require("node:http");
15
+ const https = require("node:https");
16
+ const path = require("node:path");
17
+ const zlib = require("node:zlib");
18
+ const { spawn, spawnSync } = require("node:child_process");
19
+
20
+ const { isAlive, releaseLock, takeLock } = require("./file-lock");
21
+ const { getStateFolder } = require("./sign-in");
22
+
23
+ /** The marketplaces a first install clones: the new install names' (`install_names.json` `new.repo`). */
24
+ const PREFETCH_REPOS = Object.freeze(["prbe-ai/probe"]);
25
+ /** The hidden word the launcher re-runs itself with, detached, to do the prefetch. */
26
+ const PREFETCH_VERB = "--probe-background-prefetch";
27
+ /** The whole prefetch: past it, what is unfinished is left to the agents. */
28
+ const PREFETCH_CAP_MS = 90 * 1000;
29
+ /** `off`: no prefetch (the launcher's own tests, whose throwaway homes it would outlive). */
30
+ const OFF_ENV = "PROBE_PLUGIN_PREFETCH";
31
+ const RECORD_NAME = "marketplaces.json";
32
+ const LOCK_NAME = "marketplaces.lock";
33
+ const FOLDER_NAME = "marketplaces";
34
+ /** A record still `running` this long after it started is a prefetch that died (its pid may be reused). */
35
+ const RUNNING_MAX_MS = PREFETCH_CAP_MS + 30 * 1000;
36
+ /** The tarball is ~2.5 MB; past this the download is refused (the CLI's own cap, `plugin_mirror`). */
37
+ const TARBALL_MAX_BYTES = 200 * 1024 * 1024;
38
+ /** A tarball connection silent this long is dropped: a firewall that drops packets never answers. */
39
+ const TARBALL_IDLE_MS = 20 * 1000;
40
+ /** Node's https ignores these; Python's urllib (the CLI's own download) honours them. */
41
+ const PROXY_ENVS = Object.freeze(["HTTPS_PROXY", "https_proxy", "ALL_PROXY", "all_proxy"]);
42
+
43
+ const PrefetchState = Object.freeze({ RUNNING: "running", DONE: "done" });
44
+
45
+ function getFile(name) {
46
+ return path.join(getStateFolder(), name);
47
+ }
48
+
49
+ /** `owner/name` as one file name: `prbe-ai-probe`. */
50
+ function getRepoFileName(repo) {
51
+ return repo.replace(/[^A-Za-z0-9._-]/g, "-");
52
+ }
53
+
54
+ function loadPrefetch() {
55
+ try {
56
+ const record = JSON.parse(fs.readFileSync(getFile(RECORD_NAME), "utf8"));
57
+ return record && typeof record === "object" ? record : null;
58
+ } catch {
59
+ return null;
60
+ }
61
+ }
62
+
63
+ function savePrefetch(record) {
64
+ const file = getFile(RECORD_NAME);
65
+ const temp = `${file}.${process.pid}`;
66
+ fs.writeFileSync(temp, JSON.stringify({ v: 1, ...record }), { mode: 0o600 });
67
+ fs.renameSync(temp, file);
68
+ }
69
+
70
+ /** Is `record` a prefetch still at work: running, its process alive, and younger than RUNNING_MAX_MS? */
71
+ function isRunning(record, nowMs = Date.now()) {
72
+ return Boolean(record && record.state === PrefetchState.RUNNING && isAlive(record.pid)
73
+ && Number.isFinite(record.started_at) && nowMs - record.started_at * 1000 < RUNNING_MAX_MS);
74
+ }
75
+
76
+ /** Start the prefetch in its own session, unless one runs: `"started" | "running" | "off"`. */
77
+ function startPrefetch({ launcher, env = process.env }) {
78
+ if ((env[OFF_ENV] || "").trim().toLowerCase() === "off") return "off";
79
+ if (isRunning(loadPrefetch())) return "running";
80
+ const child = spawn(process.execPath, [launcher, PREFETCH_VERB], { detached: true, stdio: "ignore", env });
81
+ child.on("error", () => {});
82
+ child.unref();
83
+ return "started";
84
+ }
85
+
86
+ /**
87
+ * The detached process's work: every repo's mirror and tarball at once, within `capMs`, then the record says
88
+ * which are here (a part unfinished at the cap is left out). A second prefetch while one runs downloads
89
+ * nothing. Returns the exit status.
90
+ */
91
+ async function runPrefetch({ repos = PREFETCH_REPOS, capMs = PREFETCH_CAP_MS, getGit = findUsableGit,
92
+ getTarballUrl = getGitHubTarballUrl } = {}) {
93
+ const lock = getFile(LOCK_NAME);
94
+ try {
95
+ if (!takeLock(lock, capMs)) return 0; // another prefetch owns them
96
+ } catch {
97
+ return 1; // no state folder to write: the agents download as before
98
+ }
99
+ const startedAt = Date.now() / 1000;
100
+ const started = performance.now();
101
+ const folder = getFile(FOLDER_NAME);
102
+ const stops = [];
103
+ const parts = Object.fromEntries(repos.map((repo) => [repo, { mirror: null, tarball: null }]));
104
+ const save = (fields) => savePrefetch({ pid: process.pid, started_at: startedAt, repos: parts, ...fields });
105
+ try {
106
+ fs.mkdirSync(folder, { recursive: true, mode: 0o700 });
107
+ removePartials(folder);
108
+ save({ state: PrefetchState.RUNNING });
109
+ const git = getGit();
110
+ const keep = (repo, kind) => (answer) => {
111
+ parts[repo][kind] = answer.file || null;
112
+ if (answer.error) parts[repo][`${kind}_error`] = answer.error;
113
+ };
114
+ const isDone = await withCap(capMs, stops, Promise.all(repos.flatMap((repo) => [
115
+ (git ? cloneMirror(git, repo, folder, stops) : Promise.resolve({ error: "no git that runs here" }))
116
+ .then(keep(repo, "mirror")),
117
+ (hasProxy() ? Promise.resolve({ error: "a proxy is set: left to the CLI, which uses it" })
118
+ : fetchTarball(getTarballUrl(repo), repo, folder, stops)).then(keep(repo, "tarball")),
119
+ ])));
120
+ save({ state: PrefetchState.DONE, finished_at: Date.now() / 1000,
121
+ seconds: Math.round((performance.now() - started) / 100) / 10,
122
+ ...(isDone ? {} : { error: "the prefetch ran out of time" }) });
123
+ return 0;
124
+ } catch (err) {
125
+ try {
126
+ save({ state: PrefetchState.DONE, finished_at: Date.now() / 1000, error: String((err && err.message) || err) });
127
+ } catch {}
128
+ return 1;
129
+ } finally {
130
+ for (const stop of stops) stop();
131
+ releaseLock(lock);
132
+ }
133
+ }
134
+
135
+ function hasProxy(env = process.env) {
136
+ return PROXY_ENVS.some((name) => (env[name] || "").trim());
137
+ }
138
+
139
+ function getGitHubTarballUrl(repo) {
140
+ return `https://codeload.github.com/${repo}/tar.gz/refs/heads/main`;
141
+ }
142
+
143
+ /** A killed prefetch's half-written parts (`<name>.<pid>`): this one holds the lock, so none is live. */
144
+ function removePartials(folder) {
145
+ for (const name of fs.readdirSync(folder)) {
146
+ if (/\.\d+$/.test(name)) fs.rmSync(path.join(folder, name), { recursive: true, force: true });
147
+ }
148
+ }
149
+
150
+ /** True once `work` ended, false when `capMs` passed first (every started part is stopped then). */
151
+ function withCap(capMs, stops, work) {
152
+ let timer;
153
+ const cap = new Promise((resolve) => {
154
+ timer = setTimeout(() => {
155
+ for (const stop of stops) stop();
156
+ resolve(false);
157
+ }, capMs);
158
+ });
159
+ return Promise.race([work.then(() => true), cap]).finally(() => clearTimeout(timer));
160
+ }
161
+
162
+ /**
163
+ * `git clone --bare` of `repo` into `<folder>/<name>.git`, swapped in whole when it finished: `{file}` (its
164
+ * name in `folder`) or `{error}`. Upload-pack filters are allowed there: Codex's clone asks `--filter=blob:none`.
165
+ */
166
+ function cloneMirror(git, repo, folder, stops) {
167
+ const name = `${getRepoFileName(repo)}.git`;
168
+ const partial = path.join(folder, `${name}.${process.pid}`);
169
+ return new Promise((resolve) => {
170
+ let child;
171
+ try {
172
+ fs.rmSync(partial, { recursive: true, force: true });
173
+ // Its own process group: the cap stops git's helpers (remote-https, index-pack) with it.
174
+ child = spawn(git, ["clone", "--bare", "--quiet", `https://github.com/${repo}.git`, partial], {
175
+ stdio: ["ignore", "ignore", "pipe"], detached: true, env: { ...process.env, GIT_TERMINAL_PROMPT: "0" },
176
+ });
177
+ } catch (err) {
178
+ return resolve({ error: `git: ${err.message}` });
179
+ }
180
+ let said = "";
181
+ let hasExited = false;
182
+ child.stderr.on("data", (chunk) => { said = (said + chunk).slice(-2000); });
183
+ stops.push(() => {
184
+ if (hasExited) return;
185
+ try { process.kill(-child.pid, "SIGKILL"); } catch {}
186
+ fs.rmSync(partial, { recursive: true, force: true });
187
+ });
188
+ child.on("error", (err) => resolve({ error: `git: ${err.message}` }));
189
+ child.on("close", (code) => {
190
+ hasExited = true;
191
+ if (code !== 0) {
192
+ fs.rmSync(partial, { recursive: true, force: true });
193
+ return resolve({ error: hideCredentials(`git clone exited ${code}: ${said.trim().split("\n").pop() || ""}`.trim()) });
194
+ }
195
+ spawnSync(git, ["-C", partial, "config", "uploadpack.allowFilter", "true"], { stdio: "ignore", timeout: 10000 });
196
+ try {
197
+ const final = path.join(folder, name);
198
+ fs.rmSync(final, { recursive: true, force: true });
199
+ fs.renameSync(partial, final);
200
+ resolve({ file: name });
201
+ } catch (err) {
202
+ fs.rmSync(partial, { recursive: true, force: true });
203
+ resolve({ error: err.message });
204
+ }
205
+ });
206
+ });
207
+ }
208
+
209
+ /** `text` with any `user:secret@` in a URL taken out (a git config may rewrite GitHub's URL to a credentialed one). */
210
+ function hideCredentials(text) {
211
+ return text.replace(/\/\/[^/@\s]+@/g, "//***@");
212
+ }
213
+
214
+ /** `repo`'s main branch as GitHub's tarball (`url`), into `<folder>/<name>.tar.gz` once it read whole and unzips. */
215
+ function fetchTarball(url, repo, folder, stops) {
216
+ const name = `${getRepoFileName(repo)}.tar.gz`;
217
+ return new Promise((resolve) => {
218
+ const request = (url.startsWith("http:") ? http : https).get(url, (response) => {
219
+ if (response.statusCode !== 200) {
220
+ response.resume();
221
+ return resolve({ error: `the tarball answered ${response.statusCode}` });
222
+ }
223
+ const chunks = [];
224
+ let size = 0;
225
+ response.on("data", (chunk) => {
226
+ size += chunk.length;
227
+ if (size > TARBALL_MAX_BYTES) request.destroy(new Error(`the tarball is over ${TARBALL_MAX_BYTES} bytes`));
228
+ else chunks.push(chunk);
229
+ });
230
+ response.on("error", (err) => resolve({ error: err.message }));
231
+ response.on("end", () => {
232
+ try {
233
+ const data = Buffer.concat(chunks);
234
+ zlib.gunzipSync(data, { maxOutputLength: TARBALL_MAX_BYTES * 4 }); // a cut-off one never reaches the CLI
235
+ const temp = path.join(folder, `${name}.${process.pid}`);
236
+ fs.writeFileSync(temp, data, { mode: 0o600 });
237
+ fs.renameSync(temp, path.join(folder, name));
238
+ resolve({ file: name });
239
+ } catch (err) {
240
+ resolve({ error: err.message });
241
+ }
242
+ });
243
+ });
244
+ request.setTimeout(TARBALL_IDLE_MS, () => request.destroy(new Error(`no answer for ${TARBALL_IDLE_MS / 1000} s`)));
245
+ request.on("error", (err) => resolve({ error: err.message }));
246
+ stops.push(() => request.destroy(new Error("stopped")));
247
+ });
248
+ }
249
+
250
+ /** The `git` this computer runs, or null: none on PATH, or macOS's stand-in with no developer tools (running
251
+ * it would put Apple's install dialog in front of the browser approval). */
252
+ function findUsableGit(env = process.env, platform = process.platform) {
253
+ const git = findOnPath("git", env);
254
+ if (!git) return null;
255
+ if (platform === "darwin" && getRealPath(git) === "/usr/bin/git") {
256
+ const tools = spawnSync("/usr/bin/xcode-select", ["-p"], { stdio: "ignore", timeout: 5000 });
257
+ if (tools.status !== 0) return null;
258
+ }
259
+ return git;
260
+ }
261
+
262
+ function getRealPath(file) {
263
+ try {
264
+ return fs.realpathSync(file);
265
+ } catch {
266
+ return file;
267
+ }
268
+ }
269
+
270
+ function findOnPath(name, env) {
271
+ for (const dir of (env.PATH || "").split(path.delimiter)) {
272
+ if (!dir) continue;
273
+ const candidate = path.join(dir, name);
274
+ try {
275
+ fs.accessSync(candidate, fs.constants.X_OK);
276
+ if (fs.statSync(candidate).isFile()) return candidate;
277
+ } catch {}
278
+ }
279
+ return null;
280
+ }
281
+
282
+ module.exports = {
283
+ FOLDER_NAME,
284
+ OFF_ENV,
285
+ PREFETCH_CAP_MS,
286
+ PREFETCH_REPOS,
287
+ PREFETCH_VERB,
288
+ PrefetchState,
289
+ RECORD_NAME,
290
+ cloneMirror,
291
+ fetchTarball,
292
+ findUsableGit,
293
+ getFile,
294
+ hideCredentials,
295
+ isRunning,
296
+ loadPrefetch,
297
+ runPrefetch,
298
+ startPrefetch,
299
+ };
@@ -6,6 +6,7 @@
6
6
 
7
7
  const { spawnSync } = require("node:child_process");
8
8
 
9
+ const { getClockMs } = require("./agent-run");
9
10
  const { getSignInBase } = require("./sign-in");
10
11
 
11
12
  /** What install.md's first step ran before: the same status, with the same flags. */
@@ -78,7 +79,8 @@ function getLastAnswer(stdout) {
78
79
 
79
80
  /**
80
81
  * Run the launcher's own `status --json --refresh` (whatever it does on this computer: answer `none` at once
81
- * with no Probe here, or run the installed CLI's status) until `deadlineMs`: `{check, stdout, code}`.
82
+ * with no Probe here, or run the installed CLI's status) until `deadlineMs` (on `agent-run.getClockMs()`):
83
+ * `{check, stdout, code}`.
82
84
  */
83
85
  function runSetupCheck({ launcher, deadlineMs, env = process.env }) {
84
86
  const ran = spawnSync(process.execPath, [launcher, ...STATUS_ARGS], {
@@ -86,7 +88,7 @@ function runSetupCheck({ launcher, deadlineMs, env = process.env }) {
86
88
  env,
87
89
  maxBuffer: MAX_ANSWER_BYTES,
88
90
  stdio: ["ignore", "pipe", "inherit"],
89
- timeout: Math.max(1000, deadlineMs - Date.now()),
91
+ timeout: Math.max(1000, deadlineMs - getClockMs()),
90
92
  });
91
93
  if (ran.error || ran.signal) return { check: SetupCheck.TIMED_OUT, stdout: "", code: 1 };
92
94
  const answer = getLastAnswer(ran.stdout);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "probe-research",
3
- "version": "0.42.0",
3
+ "version": "0.43.0",
4
4
  "description": "Set up Probe, experiment tracking for ML teams and their coding agents, from zero with npx.",
5
5
  "bin": {
6
6
  "probe-research": "bin/probe-research.js"
@@ -37,6 +37,6 @@
37
37
  "agent-swarms"
38
38
  ],
39
39
  "scripts": {
40
- "test": "node --test test/resolve.test.js test/sign-in.test.js test/launcher/launcher.test.js test/contract.test.js test/setup-check.test.js test/package.test.js"
40
+ "test": "node --test test/resolve.test.js test/sign-in.test.js test/launcher/launcher.test.js test/launcher/agent-run.test.js test/contract.test.js test/setup-check.test.js test/package.test.js test/plugin-prefetch.test.js"
41
41
  }
42
42
  }