sourcemonkey 0.0.0-stage → 0.5.0

package/lib/harness/download-watch.d.ts

@@ -0,0 +1,21 @@
+import type { Browser } from '@playwright/test';
+import { type ControlledTab } from './console-commands.js';
+export interface DownloadWatch {
+    /** Stop listening; no line prints after this. */
+    stop(): Promise<void>;
+}
+/**
+ * Start reporting the run's downloads: each through `said`, with the
+ * number of the tab it came from (the run logs it as an event), and
+ * a failure to watch at all through `out`. `tabs` is read as each
+ * download starts, so a tab opened later counts.
+ *
+ * With `downloadsDir` (a launched browser), downloads are saved there.
+ * Without, they keep the browser's own setting, as its person set it.
+ *
+ * Never throws: reporting downloads is a side feature, so a browser
+ * that refuses it gets a note, and the run goes on without it.
+ */
+export declare function watchDownloads(browser: Browser, tabs: () => ControlledTab[], said: (tab: number, text: string) => void, out: (line: string) => void, { downloadsDir }?: {
+    downloadsDir?: string;
+}): Promise<DownloadWatch>;

package/lib/harness/download-watch.js

@@ -0,0 +1,156 @@
+// Says where a download from one of the run's tabs went, as it finishes.
+// A page's own downloads (a link, a script's `<a download>` click) pass
+// SourceMonkey by, so without this a run has no record of them at all.
+// An http(s) `GM_download` gets its own line from the extension's report;
+// a `blob:` or `data:` one is an `<a download>` click in the page, so it
+// gets this line.
+//
+// Chrome reports downloads over CDP only to a client that asks
+// (`Browser.setDownloadBehavior` with `eventsEnabled`), and to all such
+// clients for all tabs, so each is checked against the run's tabs.
+import { withoutQuery } from './console-commands.js';
+/**
+ * Start reporting the run's downloads: each through `said`, with the
+ * number of the tab it came from (the run logs it as an event), and
+ * a failure to watch at all through `out`. `tabs` is read as each
+ * download starts, so a tab opened later counts.
+ *
+ * With `downloadsDir` (a launched browser), downloads are saved there.
+ * Without, they keep the browser's own setting, as its person set it.
+ *
+ * Never throws: reporting downloads is a side feature, so a browser
+ * that refuses it gets a note, and the run goes on without it.
+ */
+export async function watchDownloads(browser, tabs, said, out, { downloadsDir } = {}) {
+    let cdp;
+    let stopped = false;
+    const say = (tab, text) => {
+        if (!stopped)
+            said(tab, text);
+    };
+    try {
+        const session = cdp = await browser.newBrowserCDPSession();
+        // Ours, by guid, from start to end; a download of another client's
+        // tab is null. A promise, since finding the tab takes round trips
+        // and a small download can finish first.
+        const started = new Map();
+        // Each target's parent or opener frame, recorded as it's created: a
+        // new tab that turns into a download closes at once, often before
+        // its download could be traced back through it. Kept for the run
+        // (a few ids per tab), since a closed one may still be asked about.
+        const parents = new Map();
+        const record = ({ targetInfo: t }) => {
+            const up = t.parentFrameId ?? t.openerFrameId ?? t.openerId;
+            if (up)
+                parents.set(t.targetId, up);
+        };
+        session.on('Target.targetCreated', record);
+        session.on('Target.targetInfoChanged', record);
+        await session.send('Target.setDiscoverTargets', { discover: true });
+        session.on('Browser.downloadWillBegin', (event) => {
+            started.set(event.guid, tabOf(session, parents, tabs(), event.frameId).then((tab) => tab
+                ? { url: event.url, tab: tab.n }
+                : null));
+        });
+        session.on('Browser.downloadProgress', (event) => {
+            if (event.state === 'inProgress')
+                return;
+            const pending = started.get(event.guid);
+            if (!pending)
+                return;
+            started.delete(event.guid);
+            void pending.then((download) => {
+                if (!download)
+                    return;
+                const from = shownUrl(download.url);
+                if (event.state === 'completed') {
+                    // Chrome doesn't promise the path on all platforms.
+                    say(download.tab, `Download saved: ${event.filePath || 'in the browser\'s download folder'} (from ${from})`);
+                }
+                else {
+                    // Chrome reports a download cancelled in its UI and one that
+                    // failed the same way.
+                    say(download.tab, `Download from ${from} did not finish: It was cancelled or failed`);
+                }
+            });
+        });
+        // `allow` into a folder of ours for a launched browser: unlike
+        // `default`, it skips Chrome's prompts for a dangerous or insecure
+        // download, which nobody could answer in a hidden browser, so the
+        // download would wait forever. It keeps the file's own name, where
+        // Playwright's `allowAndName` (which its connection sets up) uses a
+        // GUID.
+        //
+        // `default` for a connected browser: its own setting, and its own
+        // prompts for a person to answer.
+        await session.send('Browser.setDownloadBehavior', downloadsDir
+            ? { behavior: 'allow', downloadPath: downloadsDir, eventsEnabled: true }
+            : { behavior: 'default', eventsEnabled: true });
+    }
+    catch (err) {
+        out(`[sm-dev] Downloads from the run's tabs won't be reported: ${err.message}`);
+        if (cdp)
+            await detach(cdp);
+        cdp = undefined;
+    }
+    return {
+        stop: async () => {
+            stopped = true;
+            if (cdp)
+                await detach(cdp);
+        },
+    };
+}
+// The run's tab a download came from, by its frame:
+// - a tab's top frame has the tab's target id;
+// - a new tab that became the download (a `target=_blank` link, a
+//   `window.open`) names the frame that opened it;
+// - a cross-site iframe is a target of its own, naming its parent frame;
+// - a same-site iframe is in a tab's frame tree.
+// Followed up a few steps, since each of those can lead to another.
+async function tabOf(cdp, parents, tabs, frameId, depth = 0) {
+    // An extension's download (`chrome.downloads`) has no frame.
+    if (!frameId || depth > 5)
+        return undefined;
+    const top = tabs.find((t) => t.targetId === frameId);
+    if (top)
+        return top;
+    const up = parents.get(frameId) ?? await cdp.send('Target.getTargetInfo', { targetId: frameId }).then(({ targetInfo: t }) => t.parentFrameId ?? t.openerFrameId ?? t.openerId, () => undefined);
+    if (up)
+        return tabOf(cdp, parents, tabs, up, depth + 1);
+    for (const tab of tabs) {
+        if (tab.page.isClosed())
+            continue;
+        const ids = await frameIds(tab).catch(() => new Set());
+        if (ids.has(frameId))
+            return tab;
+    }
+    return undefined;
+}
+async function frameIds(tab) {
+    const session = await tab.page.context().newCDPSession(tab.page);
+    try {
+        const { frameTree } = await session.send('Page.getFrameTree');
+        const ids = new Set();
+        const walk = (node) => {
+            ids.add(node.frame.id);
+            node.childFrames?.forEach(walk);
+        };
+        walk(frameTree);
+        return ids;
+    }
+    finally {
+        await detach(session);
+    }
+}
+// A `data:` URL can be megabytes, so only its kind is shown.
+function shownUrl(url) {
+    if (url.startsWith('data:'))
+        return 'a data: URL';
+    if (url.startsWith('blob:'))
+        return url;
+    return withoutQuery(url);
+}
+async function detach(session) {
+    await session.detach().catch(() => undefined);
+}

package/lib/harness/extension-backend.d.ts

@@ -0,0 +1,41 @@
+import type { BrowserContext, Worker } from '@playwright/test';
+export interface PushResult {
+    scriptId: string;
+    collectionId: string;
+    status: 'ok' | 'invalid';
+    errorMessage?: string;
+    errorLocation?: {
+        file: string;
+        line?: number;
+        column?: number;
+    };
+    replaced: boolean;
+}
+export interface LogEntry {
+    timestamp: number;
+    level: string;
+    source: string;
+    message: string;
+    tabId?: number;
+}
+export declare function extensionIdFromKey(base64Key: string): string;
+export declare function sourceMonkeyExtensionId(): Promise<string>;
+/** The origin of the extension's pages and worker: `chrome-extension://<id>`. */
+export declare function sourceMonkeyOrigin(): Promise<string>;
+/**
+ * Whether `url` is the extension's service worker. Matched by origin,
+ * not path: an extension has one worker, and its path differs between
+ * builds (it moved into extension/), so a browser's older build is still
+ * found and told it is too old rather than timing out.
+ */
+export declare function isSourceMonkeyWorker(url: string, origin: string): boolean;
+/** `findExtensionWorker` finding no SourceMonkey in the browser. */
+export declare class ExtensionMissingError extends Error {
+}
+/**
+ * The way out of a browser the run connects to, for advice: a temp
+ * browser is the default, so it's the mode flag to drop rather than
+ * `--temp-browser` to add, which the line would refuse beside it.
+ */
+export declare const DROP_BROWSER_MODE = "drop `--existing-browser` or `--open-browser` to use a temp browser";
+export declare function findExtensionWorker(ctx: BrowserContext, timeoutMs?: number): Promise<Worker>;

package/lib/harness/extension-backend.js

@@ -0,0 +1,100 @@
+// Finding the real SourceMonkey in sm-dev's browser: whether it is
+// installed and turned on, and its service worker, which every call
+// into it goes through (extension-link.ts). Playwright exposes the
+// worker on a CDP-connected browser as well as a launched one, so the
+// same `worker.evaluate` reaches it either way.
+import crypto from 'node:crypto';
+import { readFile } from 'node:fs/promises';
+// SourceMonkey's extension id. The manifest carries a fixed `key`, so
+// Chrome derives the same id for every install of it, unpacked or not:
+// the SHA-256 of the key bytes, first sixteen bytes, each hex digit
+// mapped to a letter. Read from the package's own manifest so a
+// changed key follows along.
+let cachedId = null;
+export function extensionIdFromKey(base64Key) {
+    const digest = crypto.createHash('sha256').update(Buffer.from(base64Key, 'base64')).digest('hex');
+    return digest.slice(0, 32).split('').map((c) => String.fromCharCode('a'.charCodeAt(0) + parseInt(c, 16))).join('');
+}
+export async function sourceMonkeyExtensionId() {
+    if (cachedId)
+        return cachedId;
+    const manifest = await readManifest();
+    if (typeof manifest.key !== 'string')
+        throw new Error('Internal error: the manifest has no key to derive the extension id from');
+    cachedId = extensionIdFromKey(manifest.key);
+    return cachedId;
+}
+/** The origin of the extension's pages and worker: `chrome-extension://<id>`. */
+export async function sourceMonkeyOrigin() {
+    return `chrome-extension://${await sourceMonkeyExtensionId()}`;
+}
+/**
+ * Whether `url` is the extension's service worker. Matched by origin,
+ * not path: an extension has one worker, and its path differs between
+ * builds (it moved into extension/), so a browser's older build is still
+ * found and told it is too old rather than timing out.
+ */
+export function isSourceMonkeyWorker(url, origin) {
+    return url.startsWith(`${origin}/`);
+}
+async function readManifest() {
+    return JSON.parse(await readFile(new URL('../manifest.json', import.meta.url), 'utf8'));
+}
+/** `findExtensionWorker` finding no SourceMonkey in the browser. */
+export class ExtensionMissingError extends Error {
+}
+/**
+ * The way out of a browser the run connects to, for advice: a temp
+ * browser is the default, so it's the mode flag to drop rather than
+ * `--temp-browser` to add, which the line would refuse beside it.
+ */
+export const DROP_BROWSER_MODE = 'drop `--existing-browser` or `--open-browser` to use a temp browser';
+const NOT_INSTALLED = 'SourceMonkey is not installed in this browser. Load it there as an unpacked extension, with Allow User Scripts on '
+    + `(see https://github.com/jshute96/SourceMonkey/blob/main/docs/harness-guide.md#setting-it-up), or ${DROP_BROWSER_MODE}, which loads it.`;
+// The extension's worker in this context, once its `self.SourceMonkey`
+// surface is there. A worker can be listed before its module has run
+// that far, most likely just after the browser starts.
+export async function findExtensionWorker(ctx, timeoutMs = 10000) {
+    const deadline = Date.now() + timeoutMs;
+    const worker = await listedWorker(ctx, timeoutMs);
+    const ready = () => worker.evaluate(() => typeof self.SourceMonkey !== 'undefined');
+    while (!await ready()) {
+        if (Date.now() > deadline)
+            throw new Error(`SourceMonkey did not respond within ${Math.round(timeoutMs / 1000)} seconds`);
+        await new Promise((r) => setTimeout(r, 50));
+    }
+    return worker;
+}
+// The extension's worker, as listed. An idle worker is not listed, so
+// when it is missing an extension page is opened to wake it: any page
+// of the extension will do, and the collections page has the least
+// side effects.
+async function listedWorker(ctx, timeoutMs) {
+    const origin = await sourceMonkeyOrigin();
+    const listed = () => ctx.serviceWorkers().find((w) => isSourceMonkeyWorker(w.url(), origin));
+    const found = listed();
+    if (found)
+        return found;
+    const page = await ctx.newPage();
+    try {
+        const deadline = Date.now() + timeoutMs;
+        // A navigation to a page of an extension that isn't installed is
+        // refused at once; that is the "not installed" answer.
+        try {
+            await page.goto(`${origin}/collections.html`);
+        }
+        catch {
+            throw new ExtensionMissingError(NOT_INSTALLED);
+        }
+        while (Date.now() < deadline) {
+            const w = listed();
+            if (w)
+                return w;
+            await new Promise((r) => setTimeout(r, 200));
+        }
+    }
+    finally {
+        await page.close().catch(() => undefined);
+    }
+    throw new Error(`SourceMonkey did not respond within ${Math.round(timeoutMs / 1000)} seconds`);
+}

package/lib/harness/extension-host.d.ts

@@ -0,0 +1,112 @@
+import type { Frame, Page } from '@playwright/test';
+import type { StackLayout } from '../composed-source.js';
+import { HarnessHost, type ConnectChoice, type MenuCommandRecord, type PageLink } from './host.js';
+import type { ConsoleContext } from './console-commands.js';
+import type { ExtensionLink } from './extension-link.js';
+/** What the host needs of a tab's attachment, to order its events. */
+export interface WatchedPage extends PageLink {
+    /** When the page's current document began (see `PageAttachment.documentStart`). */
+    documentStart(): number;
+    /** The tab's page, whose frames the extension's are matched to. */
+    readonly page: Page;
+}
+export declare class ExtensionHost extends HarnessHost {
+    private readonly link;
+    /**
+     * Logs a Log entry of one of the run's scripts (`scriptId`: the
+     * run's id), with the tab it was made in, if any.
+     */
+    private readonly printLog;
+    private readonly extensionIds;
+    private readonly runIds;
+    private readonly layouts;
+    private readonly tabs;
+    private readonly tabsByPage;
+    private readonly early;
+    private readonly others;
+    private readonly otherLabels;
+    private readonly logSeen;
+    private readonly liveFrom;
+    private readonly notificationIds;
+    constructor(link: ExtensionLink, 
+    /**
+     * Logs a Log entry of one of the run's scripts (`scriptId`: the
+     * run's id), with the tab it was made in, if any.
+     */
+    printLog: (entry: {
+        scriptId: string;
+        name: string;
+        level: string;
+        message: string;
+        page?: PageLink;
+    }) => void);
+    /** The run's script `runId` is the extension's `extensionId`. */
+    setPushed(runId: string, extensionId: string): void;
+    /**
+     * After the extension's worker restarted, logs the Log entries of the
+     * run's scripts made meanwhile, whose events were lost.
+     */
+    catchUpLogs(): Promise<void>;
+    private logEntry;
+    /** The extension's id for the run's script `runId`, once pushed. */
+    extensionIdOf(runId: string): string | undefined;
+    /** How the extension's script `extensionId` maps back to its files, if known. */
+    setLayout(extensionId: string, layout: StackLayout | null): void;
+    remapStack(text: string): string;
+    /**
+     * The run's script whose code one of `urls` is (a stack frame's, or
+     * a stack's text), by the entry URLs the extension runs them under.
+     */
+    scriptOf(urls: string[]): string | undefined;
+    /**
+     * Under another manager (Violentmonkey, Tampermonkey), which reports
+     * nothing: how its code's URLs name scripts, by the manager's id
+     * (`setPushed`), what every URL it runs code under starts with, and
+     * how it maps stack lines it moved.
+     */
+    external?: {
+        idOf(url: string): string | undefined;
+        prefix: string;
+        remap?(text: string): string;
+    };
+    get getsReports(): boolean;
+    /**
+     * `page` is the run's tab `tabId`. `loaded`: the tab already shows the
+     * document its first events are from (a tab adopted after it loaded),
+     * so they needn't wait for a navigation. `scriptsOff`: its claim runs
+     * no scripts (opened from a `--noscript` tab), to keep if restored.
+     */
+    addTab(page: WatchedPage, tabId: number, loaded?: boolean, scriptsOff?: boolean): void;
+    /** A tab the run will set up: its events wait for `addTab`. */
+    expectTab(tabId: number): void;
+    tabIdOf(page: PageLink): number | undefined;
+    liveFrameDocuments(page: PageLink): Promise<Map<string, Frame> | null>;
+    private matchFrames;
+    private frameNumberOf;
+    detachPage(page: PageLink): void;
+    evalHook(scriptId: string): {
+        key: string;
+        id: string;
+    } | null;
+    valuesOf(scriptId: string): Promise<Record<string, unknown>>;
+    setValues(scriptId: string, values: Record<string, unknown>): Promise<void>;
+    private extensionValues;
+    private writeExtensionValues;
+    private other;
+    /**
+     * The SourceMonkey-managed scripts, for the commands (`ConsoleContext.others`).
+     * `guarded`: `set` on one changes the person's real data.
+     */
+    otherScripts(guarded: boolean): NonNullable<ConsoleContext['others']>;
+    answerConnectPrompt(promptId: string, choice: ConnectChoice): Promise<boolean>;
+    clickNotification(requestId: string): Promise<boolean>;
+    closeNotification(requestId: string): Promise<boolean>;
+    runManualScript(page: PageLink, scriptId: string): Promise<void>;
+    fireMenuEntry(entry: MenuCommandRecord): Promise<boolean>;
+    private ingest;
+    private enqueue;
+    private handleTabEvent;
+    private recordOpenedTab;
+    private recordConnectPrompt;
+    private recordActivity;
+}