probe-research 0.42.0 → 0.44.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -34,6 +34,7 @@ const path = require("node:path");
34
34
  const agentRun = require("../lib/agent-run");
35
35
  const approval = require("../lib/approval-watch");
36
36
  const download = require("../lib/background-download");
37
+ const pluginPrefetch = require("../lib/plugin-prefetch");
37
38
  const setupCheck = require("../lib/setup-check");
38
39
  const signIn = require("../lib/sign-in");
39
40
  /** Probe's own upkeep (`PROBE_UPKEEP`), as `make sync-probe-upkeep` writes it from the CLI's flags. */
@@ -157,8 +158,10 @@ const FIX_MIN = "0.385.0";
157
158
  * 0.525.0 since install friction notes PR1: `feedback install`, the install's note to the Probe team on the
158
159
  * person's Send (`isPromptCall`); an older CLI answers "No such command 'install'". 0.524.0 was cut before it
159
160
  * merged. Launcher 0.41.0 carries it.
161
+ * 0.599.0 since install one-go: `status --check`, `import review --background`/`--renews` and per-source consent.
162
+ * Launcher 0.44.0 carries it.
160
163
  */
161
- const AGENT_MIN_CLI = "0.525.0";
164
+ const AGENT_MIN_CLI = "0.599.0";
162
165
  /** `install` flags only AGENT_MIN_CLI knows (D36). */
163
166
  const AGENT_INSTALL_FLAGS = Object.freeze(["--recommended", "--who-records", "--session-default", "--session-control",
164
167
  "--profile", "--attempt"]);
@@ -625,9 +628,10 @@ function readGlibc() {
625
628
  const REPAIR_NEEDS_NETWORK = "probe repair needs the network to fetch a working CLI";
626
629
 
627
630
  async function main() {
628
- const startedMs = Date.now();
631
+ const startedMs = agentRun.getClockMs(); // the budget's clock: pauses in sleep (plan I-F1)
629
632
  const args = process.argv.slice(2);
630
633
  if (args[0] === download.DOWNLOAD_VERB) return runBackgroundDownload();
634
+ if (args[0] === pluginPrefetch.PREFETCH_VERB) process.exit(await pluginPrefetch.runPrefetch());
631
635
  if (args[0] === EVENT_VERB) return sendBackgroundEvent(args.slice(1));
632
636
  if (args[0] === LauncherVerb.SIGN_IN) return runSignIn(args.slice(1), startedMs);
633
637
  // `npx probe-research` with no arguments runs the wizard — that is the entire
@@ -758,13 +762,16 @@ const APPROVAL_READ_RESERVE_MS = signIn.APPROVAL_TIMEOUT_MS + 1000;
758
762
  * so the agent runs it again instead (#4661 review: 15 s for the start and the exchange, as before).
759
763
  */
760
764
  const APPROVED_RESERVE_MS = 30 * 1000 + APPROVAL_READ_RESERVE_MS;
765
+ /** The first download's extras: the CLI's (`all`) and the daemon's AI libraries (plan I-F2). */
766
+ const FIRST_DOWNLOAD_EXTRAS = "all,daemon";
761
767
  /** How often a running download is looked at again. */
762
768
  const DOWNLOAD_POLL_MS = 1000;
763
769
  /** The uninstall barrier (`probe.shared.updates.uninstall_barrier.path`). */
764
770
  const BARRIER_FILE = [".local", "state", "probe", "uninstall", "barrier.json"];
765
771
 
766
- /** `status` flags only AGENT_MIN_CLI knows: the re-paste menu's answer (D84), the past-work offer's (PR24). */
767
- const AGENT_STATUS_FLAGS = Object.freeze(["--repaste-choice", "--past-work-choice"]);
772
+ /** `status` flags only AGENT_MIN_CLI knows: the re-paste menu's answer (D84), the past-work offer's (PR24), the
773
+ * install check (install one-go). */
774
+ const AGENT_STATUS_FLAGS = Object.freeze(["--repaste-choice", "--past-work-choice", "--check"]);
768
775
  /** `import` verbs only AGENT_MIN_CLI has: bringing in past work (D49, D74). `import wandb` is older. */
769
776
  const AGENT_IMPORT_VERBS = Object.freeze(["folders", "review", "start", "status", "resume"]);
770
777
  /** `session` verbs only AGENT_MIN_CLI takes from an agent: this chat's switch (T3), the new-chat default (D102), auto mode. */
@@ -868,13 +875,19 @@ async function runAgentCall(cmd, cmdArgs, startedMs, step, watch = null) {
868
875
  process.exit(ran.code);
869
876
  }
870
877
 
871
- /** The newest release's `probe` arguments for uv, pinned when the manifest names it. */
872
- async function getLatestSpec() {
878
+ /**
879
+ * The newest release's `probe` arguments for uv, pinned when the manifest names it. With `daemon` (the first
880
+ * download's): the daemon's AI libraries too (+53 MB, measured 11-15), fetched behind the browser approval, so the
881
+ * install's who-records step never builds them (~70 s) mid-install (plan I-F2). The CLI's install from this copy
882
+ * keeps the extra (`updater._dist`).
883
+ */
884
+ async function getLatestSpec({ daemon = false } = {}) {
873
885
  const latest = await latestVersion();
874
886
  const missingBuild = getMissingBuild(process.platform, process.arch,
875
887
  process.platform === "linux" ? readGlibc() : null);
876
888
  const pinned = !missingBuild && latest && atLeast(latest, AGENT_MIN_CLI) ? latest : null;
877
- return { pinned, spec: pinned ? `${DIST}[all]==${pinned}` : `${DIST}[all]>=${AGENT_MIN_CLI}`, missingBuild };
889
+ const dist = `${DIST}[${daemon ? FIRST_DOWNLOAD_EXTRAS : "all"}]`;
890
+ return { pinned, spec: pinned ? `${dist}==${pinned}` : `${dist}>=${AGENT_MIN_CLI}`, missingBuild };
878
891
  }
879
892
 
880
893
  /** A uv this launcher runs: the one on PATH when it is new enough, else the pinned one installed. */
@@ -895,7 +908,7 @@ function startBackgroundDownload() {
895
908
  * own (an old uv that runs no `python` from a tool env): the second downloads too.
896
909
  */
897
910
  async function runBackgroundDownload() {
898
- const { pinned, spec, missingBuild } = await getLatestSpec();
911
+ const { pinned, spec, missingBuild } = await getLatestSpec({ daemon: true });
899
912
  const refresh = getRefreshFlags({ repair: false, missingBuild, pinned });
900
913
  const uv = getUv();
901
914
  process.exit(download.runDownload([
@@ -914,7 +927,7 @@ async function finishStoppedUninstall(deadlineMs) {
914
927
  const refresh = getRefreshFlags({ repair: true });
915
928
  spawnSync(uv, ["tool", "run", "--isolated", "--python", PYTHON_REQUEST, ...refresh, "--from", spec, "probe",
916
929
  "repair", "--json"],
917
- { stdio: ["ignore", 2, 2], timeout: Math.max(1000, deadlineMs - WAIT_RESERVE_MS - Date.now()) });
930
+ { stdio: ["ignore", 2, 2], timeout: Math.max(1000, deadlineMs - WAIT_RESERVE_MS - agentRun.getClockMs()) });
918
931
  if (hasBarrier()) {
919
932
  await refuseAgent(AgentStep.SIGN_IN, agentRun.Reason.UNINSTALL_RUNNING,
920
933
  "An uninstall of Probe stopped halfway and is being finished first: run the same command again.",
@@ -932,7 +945,7 @@ async function waitForRunningDownload(startedMs, step, watch = null) {
932
945
  await updateWatch(watch, step, isRunning ? record.stage || download.DownloadStage.DOWNLOADING : null);
933
946
  if (!isRunning) return record;
934
947
  // Not approved yet: the approval is read on to the budget's end, the download with it (fixes-1108b).
935
- if (Date.now() >= (watch && watch.isPending() ? getWatchEndMs(startedMs) : until)) {
948
+ if (agentRun.getClockMs() >= (watch && watch.isPending() ? getWatchEndMs(startedMs) : until)) {
936
949
  await refuseUnfinished(step, watch, record.stage || download.DownloadStage.DOWNLOADING);
937
950
  }
938
951
  await sleep(DOWNLOAD_POLL_MS);
@@ -950,7 +963,7 @@ function getWatchEndMs(startedMs) {
950
963
  * that comes meanwhile while Probe still installs is answered at once, as PR39b's.
951
964
  */
952
965
  async function watchUntilBudgetEnd(watch, startedMs, step, stage) {
953
- while (watch && watch.isPending() && Date.now() + DOWNLOAD_POLL_MS < getWatchEndMs(startedMs)) {
966
+ while (watch && watch.isPending() && agentRun.getClockMs() + DOWNLOAD_POLL_MS < getWatchEndMs(startedMs)) {
954
967
  await sleep(DOWNLOAD_POLL_MS);
955
968
  await updateWatch(watch, step, stage);
956
969
  }
@@ -1019,6 +1032,10 @@ async function runSignIn(rest, startedMs) {
1019
1032
  const started = await signIn.startSignIn({ noBrowser: rest.includes("--no-browser") });
1020
1033
  if (!started.ok) await refuseAgent(AgentStep.SIGN_IN, agentRun.Reason.INCOMPLETE, started.message, 1);
1021
1034
  const downloading = findProbe(AGENT_MIN_CLI) ? "not_needed" : startBackgroundDownload();
1035
+ // The agents' plugin downloads too, so the install's agents stage only links them (plan I-F3).
1036
+ try {
1037
+ pluginPrefetch.startPrefetch({ launcher: __filename });
1038
+ } catch {} // without it the agents download as before
1022
1039
  sendEventInBackground(process.env, SIGN_IN_STARTED_EVENT, { chat_agent: signIn.detectAgent(process.env) });
1023
1040
  if (!rest.includes("--json")) {
1024
1041
  agentRun.writeLine(`Approve this computer in the browser: ${started.verification_uri_complete}`);
@@ -1050,7 +1067,7 @@ async function findDownloadedProbe(done, startedMs, step, watch = null) {
1050
1067
  const until = startedMs + agentRun.AGENT_BUDGET_MS - WAIT_RESERVE_MS;
1051
1068
  for (;;) {
1052
1069
  const probe = findProbe(AGENT_MIN_CLI);
1053
- if (probe || !done || done.state !== download.DownloadState.DONE || Date.now() >= until) return probe;
1070
+ if (probe || !done || done.state !== download.DownloadState.DONE || agentRun.getClockMs() >= until) return probe;
1054
1071
  await updateWatch(watch, step, approval.InstallStage.STARTING);
1055
1072
  await sleep(DOWNLOAD_POLL_MS);
1056
1073
  }
@@ -1085,7 +1102,7 @@ async function waitForSignIn(json, startedMs) {
1085
1102
  * answer `waiting_for_approval` the moment it started. */
1086
1103
  async function sayInstalled(watch, startedMs) {
1087
1104
  await updateWatch(watch, AgentStep.SIGN_IN_WAIT, approval.InstallStage.DONE);
1088
- if (startedMs + agentRun.AGENT_BUDGET_MS - Date.now() >= APPROVED_RESERVE_MS) return;
1105
+ if (startedMs + agentRun.AGENT_BUDGET_MS - agentRun.getClockMs() >= APPROVED_RESERVE_MS) return;
1089
1106
  await watchUntilBudgetEnd(watch, startedMs, AgentStep.SIGN_IN_WAIT, approval.InstallStage.DONE);
1090
1107
  if (watch.isPending()) {
1091
1108
  await refuseAgent(AgentStep.SIGN_IN_WAIT, agentRun.Reason.WAITING_FOR_APPROVAL,
@@ -1094,9 +1111,11 @@ async function sayInstalled(watch, startedMs) {
1094
1111
  }
1095
1112
  if (!watch.isApproved()) return;
1096
1113
  watch.markSaid(approval.InstallStage.DONE);
1114
+ // Said as the next step, never as a time limit: "too little time left" read as a failure (11-15).
1115
+ const facts = watch.getRunAgainFacts(approval.InstallStage.DONE);
1097
1116
  await refuseAgent(AgentStep.SIGN_IN_WAIT, agentRun.Reason.RUN_AGAIN,
1098
- "Probe is installed, with too little time left to save the sign-in: run the same command again.",
1099
- agentRun.EXIT_AGAIN, watch.getRunAgainFacts(approval.InstallStage.DONE));
1117
+ "You're approved and Probe is installed: run the same command once more to finish signing in.",
1118
+ agentRun.EXIT_AGAIN, { ...facts, say: `${facts.say} Finishing your sign-in next.` });
1100
1119
  }
1101
1120
 
1102
1121
  /** A `probe` at the agent floor: the one here, else the first download's (started if none runs), waited
package/lib/agent-run.js CHANGED
@@ -1,9 +1,11 @@
1
1
  /**
2
2
  * An agent's command, kept inside its 110 s (plan D3, D67): the CLI stops itself between steps
3
- * at the deadline it is handed (`PROBE_AGENT_DEADLINE`); this is only the backstop for a step that
4
- * hangs. The child runs in its own process group so the whole group can be stopped: SIGTERM at the
5
- * deadline, SIGKILL 5 s later, then one JSON line saying to run it again. Never the interactive
6
- * wizard: a detached group loses the terminal.
3
+ * at the seconds left it is handed (`PROBE_AGENT_SECONDS_LEFT`); this is only the backstop for a step
4
+ * that hangs. The 110 s count on `getClockMs()`, which pauses while the computer sleeps (plan I-F1).
5
+ *
6
+ * The child runs in its own process group so the whole group can be stopped: SIGTERM at the deadline,
7
+ * SIGKILL 5 s later, then one JSON line saying to run it again. Never the interactive wizard: a detached
8
+ * group loses the terminal.
7
9
  *
8
10
  * One output rule for every agent command (D97): JSON lines on stdout, the last one the answer. The
9
11
  * launcher's own answer is the CLI's `done` event, `{v, event: "done", ok, reason, exit_code, ...}`, its
@@ -12,6 +14,7 @@
12
14
 
13
15
  const fs = require("node:fs");
14
16
  const { spawn } = require("node:child_process");
17
+ const { performance } = require("node:perf_hooks");
15
18
 
16
19
  /** The whole command's time, the download wait included. */
17
20
  const AGENT_BUDGET_MS = 110 * 1000;
@@ -19,7 +22,9 @@ const AGENT_BUDGET_MS = 110 * 1000;
19
22
  const TEST_BUDGET_ENV = "PROBE_LAUNCHER_TEST_BUDGET_S";
20
23
  /** After SIGTERM, this long before SIGKILL. */
21
24
  const KILL_GRACE_MS = 5 * 1000;
22
- /** The variable the CLI reads its deadline from (epoch seconds; `agent_install.budget.DEADLINE_ENV`). */
25
+ /** The seconds the CLI has left when it starts (`foundation.agent_time_left.SECONDS_LEFT_ENV`). */
26
+ const SECONDS_LEFT_ENV = "PROBE_AGENT_SECONDS_LEFT";
27
+ /** The same end on the wall clock (epoch seconds), for CLIs older than SECONDS_LEFT_ENV (`DEADLINE_ENV`). */
23
28
  const DEADLINE_ENV = "PROBE_AGENT_DEADLINE";
24
29
  /** The CLI's own reserve: it is told a deadline this much before the backstop's. */
25
30
  const CLI_MARGIN_MS = 3 * 1000;
@@ -79,16 +84,28 @@ function refuse(reason, message, code, extra = {}) {
79
84
  answer({ ok: false, reason, message, ...extra }, code);
80
85
  }
81
86
 
87
+ /** The budget's clock, in whole ms (spawnSync's `timeout` takes integers only): it pauses while the computer
88
+ * sleeps and never moves with the wall clock (a Mac asleep mid-install spent its 110 s on it: plan I-F1). */
89
+ function getClockMs() {
90
+ return Math.round(performance.now());
91
+ }
92
+
93
+ /** What a child is told it has left, `leftMs`: seconds left, and the same end on the wall clock for an old CLI. */
94
+ function toTimeLeftEnv(leftMs) {
95
+ const left = Math.max(0, leftMs);
96
+ return { [SECONDS_LEFT_ENV]: String(left / 1000), [DEADLINE_ENV]: String((Date.now() + left) / 1000) };
97
+ }
98
+
82
99
  /**
83
- * Run `cmd args` with the terminal's stdio, in its own process group, until `deadlineMs` (epoch
84
- * ms): the child's own exit status, or the backstop's `run_again` when it had to be stopped.
100
+ * Run `cmd args` with the terminal's stdio, in its own process group, until `deadlineMs` (on
101
+ * `getClockMs()`): the child's own exit status, or the backstop's `run_again` when it had to be stopped.
85
102
  */
86
103
  function runWithBackstop(cmd, args, { deadlineMs, env = process.env } = {}) {
87
104
  return new Promise((resolve) => {
88
105
  const child = spawn(cmd, args, {
89
106
  stdio: "inherit",
90
107
  detached: true,
91
- env: { ...env, [DEADLINE_ENV]: String((deadlineMs - CLI_MARGIN_MS) / 1000) },
108
+ env: { ...env, ...toTimeLeftEnv(deadlineMs - CLI_MARGIN_MS - getClockMs()) },
92
109
  });
93
110
  let stopped = false;
94
111
  const signalGroup = (signal) => {
@@ -105,7 +122,7 @@ function runWithBackstop(cmd, args, { deadlineMs, env = process.env } = {}) {
105
122
  stopped = true;
106
123
  signalGroup("SIGTERM");
107
124
  killTimer = setTimeout(() => signalGroup("SIGKILL"), KILL_GRACE_MS);
108
- }, Math.max(0, deadlineMs - Date.now()));
125
+ }, Math.max(0, deadlineMs - getClockMs()));
109
126
  child.on("error", (err) => {
110
127
  clearTimeout(stopTimer);
111
128
  for (const [signal, handler] of Object.entries(handlers)) process.off(signal, handler);
@@ -136,10 +153,13 @@ module.exports = {
136
153
  KILL_GRACE_MS,
137
154
  Reason,
138
155
  SCHEMA_VERSION,
156
+ SECONDS_LEFT_ENV,
139
157
  answer,
140
158
  answerRead,
159
+ getClockMs,
141
160
  refuse,
142
161
  runWithBackstop,
143
162
  toProgressLine,
163
+ toTimeLeftEnv,
144
164
  writeLine,
145
165
  };
@@ -15,8 +15,9 @@ const fs = require("node:fs");
15
15
  const path = require("node:path");
16
16
  const { spawn, spawnSync } = require("node:child_process");
17
17
 
18
- const { AGENT_BUDGET_MS, DEADLINE_ENV } = require("./agent-run");
18
+ const { AGENT_BUDGET_MS, getClockMs, toTimeLeftEnv } = require("./agent-run");
19
19
  const { getStateFolder } = require("./sign-in");
20
+ const { isAlive, readLockHolder, releaseLock, takeLock } = require("./file-lock");
20
21
 
21
22
  const RECORD_NAME = "download.json";
22
23
  const LOG_NAME = "download.log";
@@ -68,56 +69,6 @@ function saveDownload(record) {
68
69
  fs.renameSync(temp, file);
69
70
  }
70
71
 
71
- function isAlive(pid) {
72
- if (!Number.isInteger(pid) || pid <= 1) return false;
73
- try {
74
- process.kill(pid, 0);
75
- return true;
76
- } catch (err) {
77
- return err && err.code === "EPERM";
78
- }
79
- }
80
-
81
- /** Take the one download's lock; false while a live download holds it (a stale one is broken). */
82
- function takeLock() {
83
- const lock = getFile(LOCK_NAME);
84
- fs.mkdirSync(getStateFolder(), { recursive: true, mode: 0o700 });
85
- for (let tries = 0; tries < 2; tries++) {
86
- try {
87
- const fd = fs.openSync(lock, "wx", 0o600);
88
- fs.writeSync(fd, String(process.pid));
89
- fs.closeSync(fd);
90
- return true;
91
- } catch {
92
- let holder = NaN;
93
- let age = Infinity;
94
- try {
95
- holder = parseInt(fs.readFileSync(lock, "utf8"), 10);
96
- age = Date.now() - fs.statSync(lock).mtimeMs;
97
- } catch {}
98
- if (isAlive(holder) && age < DOWNLOAD_CAP_MS) return false;
99
- try {
100
- fs.unlinkSync(lock);
101
- } catch {}
102
- }
103
- }
104
- return false;
105
- }
106
-
107
- function readLockHolder() {
108
- try {
109
- return fs.readFileSync(getFile(LOCK_NAME), "utf8");
110
- } catch {
111
- return "";
112
- }
113
- }
114
-
115
- function releaseLock() {
116
- try {
117
- if (parseInt(fs.readFileSync(getFile(LOCK_NAME), "utf8"), 10) === process.pid) fs.unlinkSync(getFile(LOCK_NAME));
118
- } catch {}
119
- }
120
-
121
72
  /**
122
73
  * Start the download in its own session, unless one runs: `"started" | "running"`. `args` are what
123
74
  * the detached launcher runs (`downloadArgs`), `launcher` its own file.
@@ -145,17 +96,18 @@ function startDownload({ launcher, args, env = process.env }) {
145
96
  * An optional step that fails is passed over (the next one does its work too). Returns the exit status.
146
97
  */
147
98
  function runDownload(steps) {
148
- if (!takeLock()) {
99
+ if (!takeLock(getFile(LOCK_NAME), DOWNLOAD_CAP_MS)) {
149
100
  // Another download runs: the record names it (the starter wrote this process's pid there).
150
- const holder = parseInt(readLockHolder(), 10);
101
+ const holder = readLockHolder(getFile(LOCK_NAME));
151
102
  if (isAlive(holder)) saveDownload({ state: DownloadState.RUNNING, pid: holder, started_at: Date.now() / 1000 });
152
103
  return 0;
153
104
  }
154
- const started = Date.now();
155
- saveDownload({ state: DownloadState.RUNNING, pid: process.pid, started_at: started / 1000 });
105
+ const startedAt = Date.now() / 1000;
106
+ const started = getClockMs();
107
+ saveDownload({ state: DownloadState.RUNNING, pid: process.pid, started_at: startedAt });
156
108
  try {
157
109
  for (const { stage, command, isOptional = false } of steps) {
158
- saveDownload({ state: DownloadState.RUNNING, stage, pid: process.pid, started_at: started / 1000 });
110
+ saveDownload({ state: DownloadState.RUNNING, stage, pid: process.pid, started_at: startedAt });
159
111
  let { ran, logged } = runStep(command, started);
160
112
  if (isOptional) continue;
161
113
  // Once more with a fresh deadline when its service never answered: uv's own download may have used up
@@ -170,18 +122,18 @@ function runDownload(steps) {
170
122
  saveDownload({ state: DownloadState.DONE, pid: process.pid, finished_at: Date.now() / 1000 });
171
123
  return 0;
172
124
  } finally {
173
- releaseLock();
125
+ releaseLock(getFile(LOCK_NAME));
174
126
  }
175
127
  }
176
128
 
177
- /** One step (`[cmd, ...args]`) within what is left of DOWNLOAD_CAP_MS: `{ran}` (null: no time left) and
178
- * where its output starts in the log (`logged`). */
129
+ /** One step (`[cmd, ...args]`) within what is left of DOWNLOAD_CAP_MS (`started`: on `getClockMs()`, which
130
+ * pauses in sleep): `{ran}` (null: no time left) and where its output starts in the log (`logged`). */
179
131
  function runStep([cmd, ...args], started) {
180
- const left = DOWNLOAD_CAP_MS - (Date.now() - started);
132
+ const left = DOWNLOAD_CAP_MS - (getClockMs() - started);
181
133
  const logged = getLogSize();
182
- // As an agent's call: the CLI's `probe` waits for a first service still starting until this deadline
134
+ // As an agent's call: the CLI's `probe` waits for a first service still starting for this long
183
135
  // (a fresh Mac's outlasted its own 10 s, so the install that runs inside it never ran: PR29).
184
- const env = { ...process.env, [DEADLINE_ENV]: String((Date.now() + AGENT_BUDGET_MS) / 1000) };
136
+ const env = { ...process.env, ...toTimeLeftEnv(AGENT_BUDGET_MS) };
185
137
  const ran = left > 0 ? spawnSync(cmd, args, { stdio: "inherit", env, timeout: left, killSignal: "SIGKILL" }) : null;
186
138
  return { ran, logged };
187
139
  }
@@ -0,0 +1,60 @@
1
+ /**
2
+ * One owner at a time for a detached launcher job (the first download, the plugin prefetch): a lock file
3
+ * holding the owner's pid, broken when that process is gone or the lock is older than the job's cap.
4
+ */
5
+
6
+ const fs = require("node:fs");
7
+ const path = require("node:path");
8
+
9
+ function isAlive(pid) {
10
+ if (!Number.isInteger(pid) || pid <= 1) return false;
11
+ try {
12
+ process.kill(pid, 0);
13
+ return true;
14
+ } catch (err) {
15
+ return err && err.code === "EPERM";
16
+ }
17
+ }
18
+
19
+ /** Take `lock` for this process; false while a live owner younger than `maxAgeMs` holds it (a stale one is broken). */
20
+ function takeLock(lock, maxAgeMs) {
21
+ fs.mkdirSync(path.dirname(lock), { recursive: true, mode: 0o700 });
22
+ for (let tries = 0; tries < 2; tries++) {
23
+ try {
24
+ const fd = fs.openSync(lock, "wx", 0o600);
25
+ fs.writeSync(fd, String(process.pid));
26
+ fs.closeSync(fd);
27
+ return true;
28
+ } catch {
29
+ let holder = NaN;
30
+ let age = Infinity;
31
+ try {
32
+ holder = parseInt(fs.readFileSync(lock, "utf8"), 10);
33
+ age = Date.now() - fs.statSync(lock).mtimeMs;
34
+ } catch {}
35
+ if (isAlive(holder) && age < maxAgeMs) return false;
36
+ try {
37
+ fs.unlinkSync(lock);
38
+ } catch {}
39
+ }
40
+ }
41
+ return false;
42
+ }
43
+
44
+ /** The pid `lock` names, or NaN. */
45
+ function readLockHolder(lock) {
46
+ try {
47
+ return parseInt(fs.readFileSync(lock, "utf8"), 10);
48
+ } catch {
49
+ return NaN;
50
+ }
51
+ }
52
+
53
+ /** Remove `lock` when this process holds it. */
54
+ function releaseLock(lock) {
55
+ try {
56
+ if (readLockHolder(lock) === process.pid) fs.unlinkSync(lock);
57
+ } catch {}
58
+ }
59
+
60
+ module.exports = { isAlive, readLockHolder, releaseLock, takeLock };
@@ -0,0 +1,299 @@
1
+ /**
2
+ * The agents' plugin downloads, behind the sign-in (plan I-F3): Probe's plugin marketplace as a bare git
3
+ * mirror (Claude Code's and Codex's first clone reads it) and its main branch's tarball (Cursor's and Kimi
4
+ * Code's copies), fetched while the person is in the browser tab. The install's agents stage then links
5
+ * them instead of downloading (37-67 s of a first install in the field, 2026-11-14).
6
+ *
7
+ * Only Probe's own folder is written: no agent's config, so a refused approval leaves the agents as they
8
+ * were. Its own process, in its own session, one at a time (`marketplaces.lock`), ended by itself at
9
+ * PREFETCH_CAP_MS; its record (`marketplaces.json`, pinned by `tests/fixtures/marketplace_prefetch.json`)
10
+ * is what the CLI reads (`shared/harness/marketplace_prefetch`). Any part that fails is left to the agent.
11
+ */
12
+
13
+ const fs = require("node:fs");
14
+ const http = require("node:http");
15
+ const https = require("node:https");
16
+ const path = require("node:path");
17
+ const zlib = require("node:zlib");
18
+ const { spawn, spawnSync } = require("node:child_process");
19
+
20
+ const { isAlive, releaseLock, takeLock } = require("./file-lock");
21
+ const { getStateFolder } = require("./sign-in");
22
+
23
+ /** The marketplaces a first install clones: the new install names' (`install_names.json` `new.repo`). */
24
+ const PREFETCH_REPOS = Object.freeze(["prbe-ai/probe"]);
25
+ /** The hidden word the launcher re-runs itself with, detached, to do the prefetch. */
26
+ const PREFETCH_VERB = "--probe-background-prefetch";
27
+ /** The whole prefetch: past it, what is unfinished is left to the agents. */
28
+ const PREFETCH_CAP_MS = 90 * 1000;
29
+ /** `off`: no prefetch (the launcher's own tests, whose throwaway homes it would outlive). */
30
+ const OFF_ENV = "PROBE_PLUGIN_PREFETCH";
31
+ const RECORD_NAME = "marketplaces.json";
32
+ const LOCK_NAME = "marketplaces.lock";
33
+ const FOLDER_NAME = "marketplaces";
34
+ /** A record still `running` this long after it started is a prefetch that died (its pid may be reused). */
35
+ const RUNNING_MAX_MS = PREFETCH_CAP_MS + 30 * 1000;
36
+ /** The tarball is ~2.5 MB; past this the download is refused (the CLI's own cap, `plugin_mirror`). */
37
+ const TARBALL_MAX_BYTES = 200 * 1024 * 1024;
38
+ /** A tarball connection silent this long is dropped: a firewall that drops packets never answers. */
39
+ const TARBALL_IDLE_MS = 20 * 1000;
40
+ /** Node's https ignores these; Python's urllib (the CLI's own download) honours them. */
41
+ const PROXY_ENVS = Object.freeze(["HTTPS_PROXY", "https_proxy", "ALL_PROXY", "all_proxy"]);
42
+
43
+ const PrefetchState = Object.freeze({ RUNNING: "running", DONE: "done" });
44
+
45
+ function getFile(name) {
46
+ return path.join(getStateFolder(), name);
47
+ }
48
+
49
+ /** `owner/name` as one file name: `prbe-ai-probe`. */
50
+ function getRepoFileName(repo) {
51
+ return repo.replace(/[^A-Za-z0-9._-]/g, "-");
52
+ }
53
+
54
+ function loadPrefetch() {
55
+ try {
56
+ const record = JSON.parse(fs.readFileSync(getFile(RECORD_NAME), "utf8"));
57
+ return record && typeof record === "object" ? record : null;
58
+ } catch {
59
+ return null;
60
+ }
61
+ }
62
+
63
+ function savePrefetch(record) {
64
+ const file = getFile(RECORD_NAME);
65
+ const temp = `${file}.${process.pid}`;
66
+ fs.writeFileSync(temp, JSON.stringify({ v: 1, ...record }), { mode: 0o600 });
67
+ fs.renameSync(temp, file);
68
+ }
69
+
70
+ /** Is `record` a prefetch still at work: running, its process alive, and younger than RUNNING_MAX_MS? */
71
+ function isRunning(record, nowMs = Date.now()) {
72
+ return Boolean(record && record.state === PrefetchState.RUNNING && isAlive(record.pid)
73
+ && Number.isFinite(record.started_at) && nowMs - record.started_at * 1000 < RUNNING_MAX_MS);
74
+ }
75
+
76
+ /** Start the prefetch in its own session, unless one runs: `"started" | "running" | "off"`. */
77
+ function startPrefetch({ launcher, env = process.env }) {
78
+ if ((env[OFF_ENV] || "").trim().toLowerCase() === "off") return "off";
79
+ if (isRunning(loadPrefetch())) return "running";
80
+ const child = spawn(process.execPath, [launcher, PREFETCH_VERB], { detached: true, stdio: "ignore", env });
81
+ child.on("error", () => {});
82
+ child.unref();
83
+ return "started";
84
+ }
85
+
86
+ /**
87
+ * The detached process's work: every repo's mirror and tarball at once, within `capMs`, then the record says
88
+ * which are here (a part unfinished at the cap is left out). A second prefetch while one runs downloads
89
+ * nothing. Returns the exit status.
90
+ */
91
+ async function runPrefetch({ repos = PREFETCH_REPOS, capMs = PREFETCH_CAP_MS, getGit = findUsableGit,
92
+ getTarballUrl = getGitHubTarballUrl } = {}) {
93
+ const lock = getFile(LOCK_NAME);
94
+ try {
95
+ if (!takeLock(lock, capMs)) return 0; // another prefetch owns them
96
+ } catch {
97
+ return 1; // no state folder to write: the agents download as before
98
+ }
99
+ const startedAt = Date.now() / 1000;
100
+ const started = performance.now();
101
+ const folder = getFile(FOLDER_NAME);
102
+ const stops = [];
103
+ const parts = Object.fromEntries(repos.map((repo) => [repo, { mirror: null, tarball: null }]));
104
+ const save = (fields) => savePrefetch({ pid: process.pid, started_at: startedAt, repos: parts, ...fields });
105
+ try {
106
+ fs.mkdirSync(folder, { recursive: true, mode: 0o700 });
107
+ removePartials(folder);
108
+ save({ state: PrefetchState.RUNNING });
109
+ const git = getGit();
110
+ const keep = (repo, kind) => (answer) => {
111
+ parts[repo][kind] = answer.file || null;
112
+ if (answer.error) parts[repo][`${kind}_error`] = answer.error;
113
+ };
114
+ const isDone = await withCap(capMs, stops, Promise.all(repos.flatMap((repo) => [
115
+ (git ? cloneMirror(git, repo, folder, stops) : Promise.resolve({ error: "no git that runs here" }))
116
+ .then(keep(repo, "mirror")),
117
+ (hasProxy() ? Promise.resolve({ error: "a proxy is set: left to the CLI, which uses it" })
118
+ : fetchTarball(getTarballUrl(repo), repo, folder, stops)).then(keep(repo, "tarball")),
119
+ ])));
120
+ save({ state: PrefetchState.DONE, finished_at: Date.now() / 1000,
121
+ seconds: Math.round((performance.now() - started) / 100) / 10,
122
+ ...(isDone ? {} : { error: "the prefetch ran out of time" }) });
123
+ return 0;
124
+ } catch (err) {
125
+ try {
126
+ save({ state: PrefetchState.DONE, finished_at: Date.now() / 1000, error: String((err && err.message) || err) });
127
+ } catch {}
128
+ return 1;
129
+ } finally {
130
+ for (const stop of stops) stop();
131
+ releaseLock(lock);
132
+ }
133
+ }
134
+
135
+ function hasProxy(env = process.env) {
136
+ return PROXY_ENVS.some((name) => (env[name] || "").trim());
137
+ }
138
+
139
+ function getGitHubTarballUrl(repo) {
140
+ return `https://codeload.github.com/${repo}/tar.gz/refs/heads/main`;
141
+ }
142
+
143
+ /** A killed prefetch's half-written parts (`<name>.<pid>`): this one holds the lock, so none is live. */
144
+ function removePartials(folder) {
145
+ for (const name of fs.readdirSync(folder)) {
146
+ if (/\.\d+$/.test(name)) fs.rmSync(path.join(folder, name), { recursive: true, force: true });
147
+ }
148
+ }
149
+
150
+ /** True once `work` ended, false when `capMs` passed first (every started part is stopped then). */
151
+ function withCap(capMs, stops, work) {
152
+ let timer;
153
+ const cap = new Promise((resolve) => {
154
+ timer = setTimeout(() => {
155
+ for (const stop of stops) stop();
156
+ resolve(false);
157
+ }, capMs);
158
+ });
159
+ return Promise.race([work.then(() => true), cap]).finally(() => clearTimeout(timer));
160
+ }
161
+
162
+ /**
163
+ * `git clone --bare` of `repo` into `<folder>/<name>.git`, swapped in whole when it finished: `{file}` (its
164
+ * name in `folder`) or `{error}`. Upload-pack filters are allowed there: Codex's clone asks `--filter=blob:none`.
165
+ */
166
+ function cloneMirror(git, repo, folder, stops) {
167
+ const name = `${getRepoFileName(repo)}.git`;
168
+ const partial = path.join(folder, `${name}.${process.pid}`);
169
+ return new Promise((resolve) => {
170
+ let child;
171
+ try {
172
+ fs.rmSync(partial, { recursive: true, force: true });
173
+ // Its own process group: the cap stops git's helpers (remote-https, index-pack) with it.
174
+ child = spawn(git, ["clone", "--bare", "--quiet", `https://github.com/${repo}.git`, partial], {
175
+ stdio: ["ignore", "ignore", "pipe"], detached: true, env: { ...process.env, GIT_TERMINAL_PROMPT: "0" },
176
+ });
177
+ } catch (err) {
178
+ return resolve({ error: `git: ${err.message}` });
179
+ }
180
+ let said = "";
181
+ let hasExited = false;
182
+ child.stderr.on("data", (chunk) => { said = (said + chunk).slice(-2000); });
183
+ stops.push(() => {
184
+ if (hasExited) return;
185
+ try { process.kill(-child.pid, "SIGKILL"); } catch {}
186
+ fs.rmSync(partial, { recursive: true, force: true });
187
+ });
188
+ child.on("error", (err) => resolve({ error: `git: ${err.message}` }));
189
+ child.on("close", (code) => {
190
+ hasExited = true;
191
+ if (code !== 0) {
192
+ fs.rmSync(partial, { recursive: true, force: true });
193
+ return resolve({ error: hideCredentials(`git clone exited ${code}: ${said.trim().split("\n").pop() || ""}`.trim()) });
194
+ }
195
+ spawnSync(git, ["-C", partial, "config", "uploadpack.allowFilter", "true"], { stdio: "ignore", timeout: 10000 });
196
+ try {
197
+ const final = path.join(folder, name);
198
+ fs.rmSync(final, { recursive: true, force: true });
199
+ fs.renameSync(partial, final);
200
+ resolve({ file: name });
201
+ } catch (err) {
202
+ fs.rmSync(partial, { recursive: true, force: true });
203
+ resolve({ error: err.message });
204
+ }
205
+ });
206
+ });
207
+ }
208
+
209
+ /** `text` with any `user:secret@` in a URL taken out (a git config may rewrite GitHub's URL to a credentialed one). */
210
+ function hideCredentials(text) {
211
+ return text.replace(/\/\/[^/@\s]+@/g, "//***@");
212
+ }
213
+
214
+ /** `repo`'s main branch as GitHub's tarball (`url`), into `<folder>/<name>.tar.gz` once it read whole and unzips. */
215
+ function fetchTarball(url, repo, folder, stops) {
216
+ const name = `${getRepoFileName(repo)}.tar.gz`;
217
+ return new Promise((resolve) => {
218
+ const request = (url.startsWith("http:") ? http : https).get(url, (response) => {
219
+ if (response.statusCode !== 200) {
220
+ response.resume();
221
+ return resolve({ error: `the tarball answered ${response.statusCode}` });
222
+ }
223
+ const chunks = [];
224
+ let size = 0;
225
+ response.on("data", (chunk) => {
226
+ size += chunk.length;
227
+ if (size > TARBALL_MAX_BYTES) request.destroy(new Error(`the tarball is over ${TARBALL_MAX_BYTES} bytes`));
228
+ else chunks.push(chunk);
229
+ });
230
+ response.on("error", (err) => resolve({ error: err.message }));
231
+ response.on("end", () => {
232
+ try {
233
+ const data = Buffer.concat(chunks);
234
+ zlib.gunzipSync(data, { maxOutputLength: TARBALL_MAX_BYTES * 4 }); // a cut-off one never reaches the CLI
235
+ const temp = path.join(folder, `${name}.${process.pid}`);
236
+ fs.writeFileSync(temp, data, { mode: 0o600 });
237
+ fs.renameSync(temp, path.join(folder, name));
238
+ resolve({ file: name });
239
+ } catch (err) {
240
+ resolve({ error: err.message });
241
+ }
242
+ });
243
+ });
244
+ request.setTimeout(TARBALL_IDLE_MS, () => request.destroy(new Error(`no answer for ${TARBALL_IDLE_MS / 1000} s`)));
245
+ request.on("error", (err) => resolve({ error: err.message }));
246
+ stops.push(() => request.destroy(new Error("stopped")));
247
+ });
248
+ }
249
+
250
+ /** The `git` this computer runs, or null: none on PATH, or macOS's stand-in with no developer tools (running
251
+ * it would put Apple's install dialog in front of the browser approval). */
252
+ function findUsableGit(env = process.env, platform = process.platform) {
253
+ const git = findOnPath("git", env);
254
+ if (!git) return null;
255
+ if (platform === "darwin" && getRealPath(git) === "/usr/bin/git") {
256
+ const tools = spawnSync("/usr/bin/xcode-select", ["-p"], { stdio: "ignore", timeout: 5000 });
257
+ if (tools.status !== 0) return null;
258
+ }
259
+ return git;
260
+ }
261
+
262
+ function getRealPath(file) {
263
+ try {
264
+ return fs.realpathSync(file);
265
+ } catch {
266
+ return file;
267
+ }
268
+ }
269
+
270
+ function findOnPath(name, env) {
271
+ for (const dir of (env.PATH || "").split(path.delimiter)) {
272
+ if (!dir) continue;
273
+ const candidate = path.join(dir, name);
274
+ try {
275
+ fs.accessSync(candidate, fs.constants.X_OK);
276
+ if (fs.statSync(candidate).isFile()) return candidate;
277
+ } catch {}
278
+ }
279
+ return null;
280
+ }
281
+
282
+ module.exports = {
283
+ FOLDER_NAME,
284
+ OFF_ENV,
285
+ PREFETCH_CAP_MS,
286
+ PREFETCH_REPOS,
287
+ PREFETCH_VERB,
288
+ PrefetchState,
289
+ RECORD_NAME,
290
+ cloneMirror,
291
+ fetchTarball,
292
+ findUsableGit,
293
+ getFile,
294
+ hideCredentials,
295
+ isRunning,
296
+ loadPrefetch,
297
+ runPrefetch,
298
+ startPrefetch,
299
+ };
@@ -6,6 +6,7 @@
6
6
 
7
7
  const { spawnSync } = require("node:child_process");
8
8
 
9
+ const { getClockMs } = require("./agent-run");
9
10
  const { getSignInBase } = require("./sign-in");
10
11
 
11
12
  /** What install.md's first step ran before: the same status, with the same flags. */
@@ -78,7 +79,8 @@ function getLastAnswer(stdout) {
78
79
 
79
80
  /**
80
81
  * Run the launcher's own `status --json --refresh` (whatever it does on this computer: answer `none` at once
81
- * with no Probe here, or run the installed CLI's status) until `deadlineMs`: `{check, stdout, code}`.
82
+ * with no Probe here, or run the installed CLI's status) until `deadlineMs` (on `agent-run.getClockMs()`):
83
+ * `{check, stdout, code}`.
82
84
  */
83
85
  function runSetupCheck({ launcher, deadlineMs, env = process.env }) {
84
86
  const ran = spawnSync(process.execPath, [launcher, ...STATUS_ARGS], {
@@ -86,7 +88,7 @@ function runSetupCheck({ launcher, deadlineMs, env = process.env }) {
86
88
  env,
87
89
  maxBuffer: MAX_ANSWER_BYTES,
88
90
  stdio: ["ignore", "pipe", "inherit"],
89
- timeout: Math.max(1000, deadlineMs - Date.now()),
91
+ timeout: Math.max(1000, deadlineMs - getClockMs()),
90
92
  });
91
93
  if (ran.error || ran.signal) return { check: SetupCheck.TIMED_OUT, stdout: "", code: 1 };
92
94
  const answer = getLastAnswer(ran.stdout);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "probe-research",
3
- "version": "0.42.0",
3
+ "version": "0.44.0",
4
4
  "description": "Set up Probe, experiment tracking for ML teams and their coding agents, from zero with npx.",
5
5
  "bin": {
6
6
  "probe-research": "bin/probe-research.js"
@@ -37,6 +37,6 @@
37
37
  "agent-swarms"
38
38
  ],
39
39
  "scripts": {
40
- "test": "node --test test/resolve.test.js test/sign-in.test.js test/launcher/launcher.test.js test/contract.test.js test/setup-check.test.js test/package.test.js"
40
+ "test": "node --test test/resolve.test.js test/sign-in.test.js test/launcher/launcher.test.js test/launcher/agent-run.test.js test/contract.test.js test/setup-check.test.js test/package.test.js test/plugin-prefetch.test.js"
41
41
  }
42
42
  }
package/upkeep.json CHANGED
@@ -38,6 +38,7 @@
38
38
  },
39
39
  "import review": {
40
40
  "--agent": 1,
41
+ "--background": 0,
41
42
  "--chats-agent": 1,
42
43
  "--chats-all": 0,
43
44
  "--chats-folder": 1,
@@ -45,6 +46,7 @@
45
46
  "--chats-since": 1,
46
47
  "--folder": 1,
47
48
  "--json": 0,
49
+ "--renews": 1,
48
50
  "--sessions": 1,
49
51
  "--skip-files-over": 1,
50
52
  "--wandb-folder": 1,
@@ -112,6 +114,7 @@
112
114
  "-y": 0
113
115
  },
114
116
  "status": {
117
+ "--check": 0,
115
118
  "--demo-choice": 1,
116
119
  "--json": 0,
117
120
  "--past-work-choice": 1,