pw-repl 0.4.0 → 0.4.1

package/README.md

@@ -6,6 +6,36 @@ one shows in the REPL, so each sees what the other does. The REPL inspects the p
 (requests, console, and each step a person takes), fakes, patches or slows responses, and emulates phones,
 dark mode, locales and timezones.
 
+## Why use it
+
+For working on a web page together with an agent, in the browser you already use:
+
+- **Your browser, as it is.** It connects to a running Chromium, with your tabs, sign-ins and state, and
+  leaves it running when it stops; nothing is reset to a fresh profile.
+- **One session, seen by everyone.** You and any number of agents drive the same browser, each with a
+  selected tab of its own, and every command shows in the REPL, whoever sent it.
+- **What you did, for an agent to read.** `watch` records each step you take in a tab, with the requests
+  it caused, so you can reproduce a bug by hand and the agent can see exactly what happened.
+- **Breaking things on purpose.** Fake, patch, delay or fail responses, cut or slow the network, emulate a
+  phone, a locale or a timezone, all per tab, so your other tabs are left alone.
+
+## How it differs from playwright-cli
+
+[playwright-cli](https://github.com/microsoft/playwright-cli) is Microsoft's command line for agents. It
+gives each agent a browser session of its own; pw-repl shares one browser between a person and agents.
+Its command names and most of their options work here too (`help playwright-cli` lists them), and where
+pw-repl differs, it is because someone else may be using the browser:
+
+- `open` and `close` open a tab and turn off your modes; they never start, close or wipe a browser.
+- Routes, network and emulation apply to one tab, not the whole browser.
+- File pickers are not caught, since they may be the person's: `upload` names the input instead.
+- There is no snapshot after every command, since a person reads the REPL too; `snapshot` shows one.
+- Instead of `-s=<session>`, an agent is a client of its own (`send -c <name>`).
+- `watch` records what a person does as steps with their requests, not as generated code.
+
+For a browser of its own per agent, isolated sessions, traces, video, PDFs or generated test code,
+playwright-cli is the better fit.
+
 ## Getting started
 
 ### Prerequisites

package/bin/pw-repl.js

@@ -132,7 +132,7 @@ function help(topic) {
   const helpText = require('../lib/help');
   const text = helpText.render(topic);
   if (text === null) {
-    console.log(`Error: No help for ${topic}. Topics: ${Object.keys(helpText.TOPICS).join(', ')}`);
+    console.log(`Error: ${helpText.notFound(topic)}`);
     process.exit(1);
   }
   console.log(text);

package/lib/commands.js

@@ -180,6 +180,28 @@ function keptSession(p) {
   return keptSessions.get(p);
 }
 
+// Chrome all but stops drawing a tab that is not in front once it has had
+// input (about a frame a second), and Playwright waits for frames to see an
+// element hold still before acting on it, so each click in a tab opened in the
+// background took 1-2s. A screencast keeps the tab drawing without bringing it
+// to the front, for as long as the REPL runs (the throttle is back within
+// seconds of stopping it). Its frames are tiny and rare; the cost is that the
+// tab renders at full rate, as if in front, an animation or a video included.
+const drawingPages = new WeakSet();
+
+async function keepDrawing(p) {
+  if (drawingPages.has(p)) return;
+  drawingPages.add(p);
+  try {
+    const session = await keptSession(p);
+    session.on('Page.screencastFrame', frame => session.send('Page.screencastFrameAck', { sessionId: frame.sessionId }).catch(() => {}));
+    await session.send('Page.startScreencast', { format: 'jpeg', quality: 1, maxWidth: 8, maxHeight: 8, everyNthFrame: 60 });
+  } catch {
+    // Only slower without it.
+    drawingPages.delete(p);
+  }
+}
+
 const networkEnabled = new WeakSet();
 // page -> how its network is changed: { offline: true }, or slowed { latency, down, up } (ms, kbps).
 const networkChanged = new WeakMap();
@@ -453,6 +475,11 @@ const BODY_TIMEOUT = 15000;
 // Hidden unless asked for: they are rarely what is being checked and crowd out
 // the API calls that are.
 const RECENT_HIDDEN_TYPES = new Set(['image', 'font', 'stylesheet', 'media']);
+// The bodies body prints; any other is binary.
+const TEXT_BODY = /json|^text\/|javascript|xml|html|x-www-form-urlencoded/;
+// A watched tab keeps each body watch shows as it arrives, since the browser
+// drops them once the tab navigates, often before anyone has read them.
+const KEPT_BODY_MAX = 100 * 1024;
 const recentLogs = new WeakMap();
 const consoleLogs = new WeakMap();
 // request -> how a route changed it (faked or patched), shown after its status.
@@ -523,6 +550,7 @@ function ensureRecentLog(p) {
       if (res) status = String(res.status());
     } catch {}
     finish(req, changedRequests.has(req) ? `${status} ${changedRequests.get(req)}` : status, res);
+    keepBody(p, requestEntries.get(req), res);
   });
   p.on('requestfailed', req => {
     finish(req, `failed: ${req.failure()?.errorText || 'unknown'}`);
@@ -564,6 +592,38 @@ const WATCH_MAX = 200;
 const WATCH_REQUESTS_SHOWN = 5;
 // Scripts too: a dev server loads dozens per navigation, crowding out the API calls.
 const WATCH_HIDDEN_TYPES = new Set([...RECENT_HIDDEN_TYPES, 'script']);
+
+// Tabs whose watch on --next-tab is still starting: the page can load before it is ready.
+const startingWatches = new WeakSet();
+
+// A page that navigates as soon as a request answers (a login that redirects)
+// takes the body with it before the request is even reported finished; told to
+// in time, the browser keeps them outside the page instead. Once per tab, from
+// its first watch, and never switched off: up to 50 MB of the browser's memory
+// per watched tab, for as long as the REPL runs.
+const DURABLE_TOTAL = 50 * 1024 * 1024;
+const durablePages = new WeakSet();
+
+async function keepBodiesDurable(p) {
+  if (durablePages.has(p)) return;
+  durablePages.add(p);
+  try {
+    const session = await keptSession(p);
+    await session.send('Network.configureDurableMessages', { maxTotalBufferSize: DURABLE_TOTAL, maxResourceBufferSize: KEPT_BODY_MAX });
+  } catch {
+    // An older Chrome: bodies are still kept once their requests finish.
+  }
+}
+
+async function keepBody(p, entry, res) {
+  if (!entry || !res || !(watches.get(p)?.on || startingWatches.has(p)) || WATCH_HIDDEN_TYPES.has(entry.type)) return;
+  const headers = res.headers();
+  if (!TEXT_BODY.test(headers['content-type'] || '') || Number(headers['content-length']) > KEPT_BODY_MAX) return;
+  try {
+    const buffer = await res.body();
+    if (buffer.length <= KEPT_BODY_MAX) entry.kept = buffer;
+  } catch {}
+}
 const TYPING_BACKDATE_MAX = 60000;
 const PAGE_ACTIONS = new Set(['click', 'check', 'uncheck', 'select', 'fill', 'type', 'press', 'submit']);
 // page -> its binding's record function; the binding is added once per tab.
@@ -810,6 +870,7 @@ function printLive(p, watch) {
 }
 
 async function startWatching(p, changes = false, live = false) {
+  await keepBodiesDurable(p);
   let watch = watches.get(p);
   if (!watch) {
     // shownSeq and shownRequestId mark how far watch new has read.
@@ -880,6 +941,11 @@ async function startWatching(p, changes = false, live = false) {
 // new tab is watched from the moment it opens, since its first page may load
 // at once; one that turns out not to be it is left as it was.
 let nextTab = null;
+
+// The sender's own --next-tab is still waiting: the tab will be selected for it.
+function waitingForMe() {
+  return !!nextTab && nextTab.client === state.client;
+}
 const BROWSER_PAGE = /^(?:about|chrome|chrome-search|devtools|edge):/;
 
 function waitingText() {
@@ -888,7 +954,10 @@ function waitingText() {
 
 function watchNewTab(p) {
   const want = nextTab;
+  startingWatches.add(p);
+  keepBodiesDurable(p);
   startWatching(p, want.changes, want.live).then(() => {
+    startingWatches.delete(p);
     watches.get(p).owner = want.client;
     const onNavigate = frame => {
       if (frame !== p.mainFrame() || BROWSER_PAGE.test(frame.url())) return;
@@ -912,12 +981,16 @@ function watchNewTab(p) {
       // Selected for the client that asked, whoever's command runs now.
       const record = clientRecord(want.client);
       if (record.page !== p) { record.previousPage = record.page; record.page = p; }
+      keepDrawing(p);
       out.notice(`Watching the new tab ${url}, now the selected tab (watch on --next-tab)`);
     };
     p.on('framenavigated', onNavigate);
     // Opened on its page already, as a link to a new tab does.
     if (p.url() && !BROWSER_PAGE.test(p.url())) onNavigate(p.mainFrame());
-  }).catch(error => out.notice(`Could not watch the new tab ${p.url()}: ${error.message}`));
+  }).catch(error => {
+    startingWatches.delete(p);
+    out.notice(`Could not watch the new tab ${p.url()}: ${error.message}`);
+  });
 }
 
 // For each step of a watch, the requests made between it and the next step.
@@ -1291,6 +1364,7 @@ async function openOwnTab() {
   if (!opened) opened = await ctx.newPage();
   openedTabs.add(opened);
   tabOwners.set(opened, state.client);
+  keepDrawing(opened);
   return opened;
 }
 
@@ -1369,6 +1443,7 @@ const commands = {
       if (p !== state.page) state.previousPage = state.page;
       state.page = p;
       ensureDialogHandler(p);
+      keepDrawing(p);
     };
     const byUrl = part => {
       const matches = all.filter(p => p.url().includes(part));
@@ -1960,19 +2035,21 @@ const commands = {
       await new Promise(resolve => setTimeout(resolve, 50));
     }
     if (!entry.response) throw new Error(`#${id} has no response${entry.status === 'pending' ? ' yet' : ` (${entry.status})`}`);
-    // Not withTimeout: its "timed out" error would disconnect the REPL, and
-    // reading a body cannot have changed anything.
-    let timer;
-    const late = new Promise((_, reject) => { timer = setTimeout(() => reject(new Error(`did not arrive within ${BODY_TIMEOUT / 1000}s`)), BODY_TIMEOUT); });
-    let buffer;
-    try { buffer = await Promise.race([entry.response.body(), late]); }
-    // Playwright's reason ends in advice for its own API ("Read response.body() before..."), so only its first line is kept.
-    catch (error) { throw new Error(`The body of #${id} is not available: ${String(error.message).split('\n')[0]} (the browser drops bodies, e.g. after the tab navigates)`); }
-    finally { clearTimeout(timer); }
+    let buffer = entry.kept;
+    if (!buffer) {
+      // Not withTimeout: its "timed out" error would disconnect the REPL, and
+      // reading a body cannot have changed anything.
+      let timer;
+      const late = new Promise((_, reject) => { timer = setTimeout(() => reject(new Error(`did not arrive within ${BODY_TIMEOUT / 1000}s`)), BODY_TIMEOUT); });
+      try { buffer = await Promise.race([entry.response.body(), late]); }
+      // Playwright's reason ends in advice for its own API ("Read response.body() before..."), so only its first line is kept.
+      catch (error) { throw new Error(`The body of #${id} is not available: ${String(error.message).split('\n')[0]} (the browser drops bodies, e.g. after the tab navigates; a watched tab keeps them as they arrive)`); }
+      finally { clearTimeout(timer); }
+    }
     const type = entry.response.headers()['content-type'] || '';
     out.log(`#${id} ${entry.method} ${entry.status} ${entry.url} (${type || 'no content type'}, ${buffer.length} bytes)`);
     if (!buffer.length) return;
-    if (!/json|^text\/|javascript|xml|html|x-www-form-urlencoded/.test(type)) { out.log('[binary body not shown]'); return; }
+    if (!TEXT_BODY.test(type)) { out.log('[binary body not shown]'); return; }
     let text = buffer.toString('utf8');
     if (/json/.test(type)) { try { text = JSON.stringify(JSON.parse(text), null, 2); } catch {} }
     printOutput(text, all);
@@ -2012,6 +2089,10 @@ const commands = {
       out.log(`${waitingText()}: it is watched from its first page, and selected. watch off stops waiting.`);
       return;
     }
+    if (sub !== 'on' && arg !== 'off' && waitingForMe() && (!state.page || state.page.isClosed())) {
+      out.log(`${waitingText()} (watch on --next-tab); it is selected once it opens, and watch then shows its steps.`);
+      return;
+    }
     if (sub === 'on' && flags.every(f => f === '--changes' || f === '--live')) {
       if (!state.page || state.page.isClosed()) throw new Error('No tab is selected; select one with tab <index|url-part>, or open one with tab new');
       const changes = flags.includes('--changes');
@@ -2175,7 +2256,7 @@ const commands = {
   async help(args) {
     const topic = (args || '').trim();
     const text = HELP.render(topic);
-    if (text === null) throw new Error(`No help for ${topic}. Topics: ${Object.keys(HELP.TOPICS).join(', ')}`);
+    if (text === null) throw new Error(HELP.notFound(topic));
     out.log(text);
   },
 
@@ -2227,9 +2308,10 @@ function watchPage(p) {
   if (nextTab && !opening) watchNewTab(p);
 }
 
-// watch on --next-tab waits for a tab, and watch off stops waiting, with none selected.
+// watch on --next-tab waits for a tab, and watch off stops waiting, with none selected;
+// watch says it is still waiting.
 function needsTab(cmd, args) {
-  return !(cmd === 'watch' && (/(?:^|\s)--next-tab(?:\s|$)/.test(args) || /^\s*off\s*$/.test(args)));
+  return !(cmd === 'watch' && (/(?:^|\s)--next-tab(?:\s|$)/.test(args) || /^\s*off\s*$/.test(args) || waitingForMe()));
 }
 
 // Tab completion for the prompt: command names first, then the fixed words

package/lib/help.js

@@ -135,7 +135,7 @@ const COMMANDS = {
   watch: {
     usage: 'watch on [--changes] [--live] | off | [n|new]',
     summary: 'record what happens in the tab; show the last n steps',
-    detail: 'Off until watch on; a watch on after watch off starts a new recording. Records clicks, typing (once\nit pauses), Enter and Escape, form changes, submits and navigations, each described by role and name\nlike snapshot, with up to 5 of the requests it caused underneath, each with its status as in requests\n(body <#> for one), leaving out scripts and what requests hides. The REPL\'s own fill is recorded\nas type.\n\nwatch on --next-tab [url-part] returns at once, then waits for the next tab someone opens (tab new,\na client\'s own, does not count), or the next whose URL contains url-part, and watches it from its\nfirst page and selects it for the client that asked: for a tab someone is about to open. It takes\n--changes and --live too; watch off stops waiting, and modes lists it.\n\nwatch on its own says whether it is on and shows the last 20 steps; watch <n> shows the last n.\nwatch new shows only the steps it has not shown yet, and requests that have since arrived for the\nlast one.\n\n--live also prints each step in the REPL window (never in the server\'s answer to a command) once it\nsettles, with its requests and changes, marked [watch], or [watch <url>] for a tab that is not the\nselected one. A step cut short by the next one, or by watch off, prints at once; changes it made\nthen show under the next step.\n\n--changes also shows what each step changed on the page once it settles, in up to 5 lines: + added,\n- removed, ~ changed, each element once with the first names inside it. Not for navigations: what a\npage changes on its own after a step settles (data arriving later) shows under the next step.\n\nA watch belongs to the tab, not to a client: every client sees the same steps, watch new\'s place is\nshared, and anyone\'s watch off stops it.\n\nTyped values are not recorded, and password fields not at all; --changes leaves out field values,\nbut what the page itself shows (e.g. "Hello <name>") is shown. Nothing is replayed.',
+    detail: 'Off until watch on; a watch on after watch off starts a new recording. Records clicks, typing (once\nit pauses), Enter and Escape, form changes, submits and navigations, each described by role and name\nlike snapshot, with up to 5 of the requests it caused underneath, each with its status as in requests\n(body <#> for one), leaving out scripts and what requests hides. The REPL\'s own fill is recorded\nas type.\n\nFrom its first watch on, a tab keeps response bodies of up to 100 KB, so body <#> still shows one\nafter the tab navigates, even when the page navigates as soon as the request answers.\n\nwatch on --next-tab [url-part] returns at once, then waits for the next tab someone opens (tab new,\na client\'s own, does not count), or the next whose URL contains url-part, and watches it from its\nfirst page and selects it for the client that asked: for a tab someone is about to open. It takes\n--changes and --live too; watch off stops waiting, and modes lists it.\n\nwatch on its own says whether it is on and shows the last 20 steps; watch <n> shows the last n.\nwatch new shows only the steps it has not shown yet, and requests that have since arrived for the\nlast one.\n\n--live also prints each step in the REPL window (never in the server\'s answer to a command) once it\nsettles, with its requests and changes, marked [watch], or [watch <url>] for a tab that is not the\nselected one. A step cut short by the next one, or by watch off, prints at once; changes it made\nthen show under the next step.\n\n--changes also shows what each step changed on the page once it settles, in up to 5 lines: + added,\n- removed, ~ changed, each element once with the first names inside it. Not for navigations: what a\npage changes on its own after a step settles (data arriving later) shows under the next step.\n\nA watch belongs to the tab, not to a client: every client sees the same steps, watch new\'s place is\nshared, and anyone\'s watch off stops it.\n\nTyped values are not recorded, and password fields not at all; --changes leaves out field values,\nbut what the page itself shows (e.g. "Hello <name>") is shown. Nothing is replayed.',
   },
   text: { usage: 'text [--all] <selector>', summary: 'visible text of the first match' },
   html: { usage: 'html [--all] <selector>', summary: 'outer HTML of the first match' },
@@ -174,7 +174,7 @@ const COMMANDS = {
   body: {
     usage: 'body [--all] <#>|<url-part>',
     summary: 'the response body of request <#>, or of the latest one matching',
-    detail: 'body <url-part> shows the latest finished request whose URL contains url-part (body slots.json).\n\nRead from the browser on demand: JSON is pretty-printed, binary is not shown, output is capped. The\nbrowser may drop a body (e.g. after the tab navigates); then body reports not available.\n\nBodies can contain sensitive data.',
+    detail: 'body <url-part> shows the latest finished request whose URL contains url-part (body slots.json).\n\nRead from the browser on demand: JSON is pretty-printed, binary is not shown, output is capped. The\nbrowser may drop a body (e.g. after the tab navigates); then body reports not available, unless a\nwatch kept it (help watch).\n\nBodies can contain sensitive data.',
   },
   console: {
     usage: 'console [--all] [n] [level|filter]',
@@ -280,4 +280,12 @@ Different on purpose, since the browser may be someone else's too:
   return null;
 }
 
-module.exports = { render, TOPICS, COMMANDS };
+// pw-repl's own commands are run at the shell, so their help is the usage.
+const SHELL_COMMANDS = new Set(['run', 'serve', 'send', 'attach', 'stop', 'where', 'skill']);
+
+function notFound(topic) {
+  if (SHELL_COMMANDS.has(topic)) return `${topic} is run at the shell (pw-repl ${topic}), not in the REPL; pw-repl --help has its options`;
+  return `No help for ${topic}. Topics: ${Object.keys(TOPICS).join(', ')}`;
+}
+
+module.exports = { render, notFound, TOPICS, COMMANDS };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pw-repl",
-  "version": "0.4.0",
+  "version": "0.4.1",
   "description": "Text REPL for driving an existing Chromium through Playwright over CDP",
   "keywords": [
     "playwright",

package/skill/SKILL.md

@@ -99,8 +99,9 @@ from its first page.
 
 When watching someone:
 
-- Read `body <#>` for any request that matters as soon as it shows up: the browser drops a body once
-  its tab navigates.
+- A watched tab keeps response bodies of up to 100 KB, so `body <#>` still has them after the tab
+  navigates. Read a bigger one that matters as soon as it shows up in `requests`: the browser drops it
+  once the tab navigates.
 - `watch` leaves out images and other static files, failed ones too; `console error` and
   `requests --all` show what failed.
 - Without a url-part, `--next-tab` takes the first tab anyone opens; check `tab` and `info` once it