sourcemonkey 0.0.0-stage → 0.5.0

package/lib/harness/manager-fetch.d.ts

@@ -0,0 +1,33 @@
+import { type ManagerName } from './managers.js';
+/** Where fetched managers are kept. */
+export declare function managerCacheDir(): string;
+/** The folder a manager is fetched into, which is loaded unpacked. */
+export declare function fetchedDir(name: ManagerName): string;
+/** The version fetched, or null when there is none. */
+export declare function fetchedVersion(name: ManagerName): string | null;
+export interface Fetched {
+    dir: string;
+    version: string;
+    /** Newly downloaded, rather than already the newest. */
+    downloaded: boolean;
+    /** The version it replaced, if any. */
+    replaced?: string;
+}
+/**
+ * Fetches the manager's newest version, unless it's fetched already.
+ * Throws an error saying which step failed.
+ */
+export declare function fetchManager(name: ManagerName, out: (line: string) => void): Promise<Fetched>;
+/**
+ * A CRX3 file's zip, and the public key (base64) its extension id comes
+ * from. Throws when it isn't one, or none of its keys gives `id`.
+ */
+export declare function readCrx(crx: Buffer, id: string): {
+    key: string;
+    zip: Buffer;
+};
+/**
+ * Writes a zip's files under `dir`: stored or deflated entries, which
+ * is what a CRX holds. A name that would land outside `dir` is refused.
+ */
+export declare function unzipTo(zip: Buffer, dir: string): Promise<void>;

package/lib/harness/manager-fetch.js

@@ -0,0 +1,239 @@
+// `manager fetch`: downloads Tampermonkey or Violentmonkey from the
+// Chrome Web Store, unpacked, into sm-dev's cache, for `manager
+// install` and a temp browser to load.
+//
+// - One folder per manager, `~/.cache/sm-dev/managers/<name>`
+//   (`$XDG_CACHE_HOME` when set), replaced whole by a newer version, so
+//   a browser that loaded it from there reloads the new one.
+// - The download is a CRX: a header, with the store's signature and
+//   public key, then a zip. The key goes into the unpacked manifest as
+//   `key`, so the copy keeps the store's extension id, as the store's
+//   own install would have.
+// - The store's update check says the newest version and the file's
+//   SHA-256, which the download is checked against.
+import { createHash, randomBytes } from 'node:crypto';
+import { existsSync, readFileSync } from 'node:fs';
+import { mkdir, rename, rm, writeFile } from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { inflateRawSync } from 'node:zlib';
+import { extensionIdFromKey } from './extension-backend.js';
+import { MANAGERS } from './managers.js';
+// The update service Chrome itself asks. `prodversion` is the Chrome
+// version asking: a high one gets the newest build.
+const UPDATE_URL = 'https://clients2.google.com/service/update2/crx';
+const PROD_VERSION = '999.0';
+/** Where fetched managers are kept. */
+export function managerCacheDir() {
+    const cache = process.env.XDG_CACHE_HOME?.trim() || path.join(os.homedir(), '.cache');
+    return path.join(cache, 'sm-dev', 'managers');
+}
+/** The folder a manager is fetched into, which is loaded unpacked. */
+export function fetchedDir(name) {
+    return path.join(managerCacheDir(), name);
+}
+/** The version fetched, or null when there is none. */
+export function fetchedVersion(name) {
+    try {
+        const manifest = JSON.parse(readFileSync(path.join(fetchedDir(name), 'manifest.json'), 'utf8'));
+        return typeof manifest.version === 'string' ? manifest.version : null;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Fetches the manager's newest version, unless it's fetched already.
+ * Throws an error saying which step failed.
+ */
+export async function fetchManager(name, out) {
+    const { label, storeId } = MANAGERS[name];
+    if (!storeId)
+        throw new Error(`Internal error: ${label} has no Web Store id to fetch it by`);
+    const dir = fetchedDir(name);
+    const have = fetchedVersion(name);
+    const latest = await newestVersion(label, storeId);
+    if (have === latest.version)
+        return { dir, version: have, downloaded: false };
+    out(`[sm-dev] Fetching ${label} ${latest.version} from the Chrome Web Store`);
+    let crx;
+    try {
+        const res = await fetch(latest.url, { signal: AbortSignal.timeout(120_000) });
+        if (!res.ok)
+            throw new Error(`HTTP ${res.status}`);
+        crx = Buffer.from(await res.arrayBuffer());
+    }
+    catch (err) {
+        throw new Error(`${label} could not be downloaded from the Chrome Web Store: ${err.message}`);
+    }
+    if (latest.sha256 && createHash('sha256').update(crx).digest('hex') !== latest.sha256) {
+        throw new Error(`The ${label} download from the Chrome Web Store doesn't match the checksum the store gave. Try again.`);
+    }
+    const { key, zip } = readCrx(crx, storeId);
+    const parent = managerCacheDir();
+    await mkdir(parent, { recursive: true });
+    // Unpacked beside it, then swapped in, so a failure leaves the old
+    // copy as it was.
+    const fresh = path.join(parent, `.${name}-${randomBytes(4).toString('hex')}`);
+    try {
+        await unzipTo(zip, fresh);
+        // The store's signature over the files, which Chrome refuses in an
+        // unpacked extension (a name starting with _ is reserved).
+        await rm(path.join(fresh, '_metadata'), { recursive: true, force: true });
+        const manifestFile = path.join(fresh, 'manifest.json');
+        const manifest = JSON.parse(readFileSync(manifestFile, 'utf8'));
+        manifest.key = key;
+        await writeFile(manifestFile, JSON.stringify(manifest, null, 2));
+        if (existsSync(dir)) {
+            const old = `${fresh}-old`;
+            await rename(dir, old);
+            await rename(fresh, dir);
+            await rm(old, { recursive: true, force: true });
+        }
+        else {
+            await rename(fresh, dir);
+        }
+    }
+    catch (err) {
+        await rm(fresh, { recursive: true, force: true });
+        throw new Error(`${label} could not be unpacked into ${dir}: ${err.message}`);
+    }
+    return { dir, version: latest.version, downloaded: true, ...(have ? { replaced: have } : {}) };
+}
+// The store's newest version of an extension, with where to download it.
+async function newestVersion(label, storeId) {
+    const query = `${UPDATE_URL}?prodversion=${PROD_VERSION}&acceptformat=crx3&x=${encodeURIComponent(`id=${storeId}&uc`)}`;
+    let xml;
+    try {
+        const res = await fetch(query, { signal: AbortSignal.timeout(30_000) });
+        if (!res.ok)
+            throw new Error(`HTTP ${res.status}`);
+        xml = await res.text();
+    }
+    catch (err) {
+        throw new Error(`The Chrome Web Store could not be asked for ${label}'s newest version: ${err.message}`);
+    }
+    const check = /<updatecheck\b([^>]*)>/.exec(xml)?.[1] ?? '';
+    const attr = (attrName) => new RegExp(`\\b${attrName}="([^"]*)"`).exec(check)?.[1];
+    const version = attr('version');
+    const url = attr('codebase');
+    if (attr('status') !== 'ok' || !version || !url) {
+        throw new Error(`The Chrome Web Store has no download for ${label} (status ${attr('status') ?? 'missing'})`);
+    }
+    return { version, url: url.replace(/&amp;/g, '&'), sha256: attr('hash_sha256') };
+}
+/**
+ * A CRX3 file's zip, and the public key (base64) its extension id comes
+ * from. Throws when it isn't one, or none of its keys gives `id`.
+ */
+export function readCrx(crx, id) {
+    if (crx.length < 12 || crx.toString('latin1', 0, 4) !== 'Cr24' || crx.readUInt32LE(4) !== 3) {
+        throw new Error('The download is not a Chrome extension package (CRX3)');
+    }
+    const headerSize = crx.readUInt32LE(8);
+    const header = crx.subarray(12, 12 + headerSize);
+    // CrxFileHeader: field 2 (sha256_with_rsa) and 3 (sha256_with_ecdsa)
+    // are AsymmetricKeyProofs, whose field 1 is the public key.
+    const keys = [];
+    for (const [field, proof] of protoFields(header)) {
+        if (field !== 2 && field !== 3)
+            continue;
+        for (const [inner, value] of protoFields(proof))
+            if (inner === 1)
+                keys.push(value);
+    }
+    const key = keys.map((k) => k.toString('base64')).find((k) => extensionIdFromKey(k) === id);
+    if (!key)
+        throw new Error(`The package has no key for extension id ${id}`);
+    return { key, zip: crx.subarray(12 + headerSize) };
+}
+// The length-delimited fields of a protobuf message, as [number, bytes].
+// Other wire types are skipped.
+function protoFields(buf) {
+    const fields = [];
+    let i = 0;
+    const varint = () => {
+        let value = 0;
+        let shift = 0;
+        for (;;) {
+            if (i >= buf.length)
+                throw new Error('The package header is cut short');
+            const byte = buf[i++];
+            value += (byte & 0x7f) * 2 ** shift;
+            if (!(byte & 0x80))
+                return value;
+            shift += 7;
+        }
+    };
+    while (i < buf.length) {
+        const tag = varint();
+        const wire = tag % 8;
+        const field = Math.floor(tag / 8);
+        if (wire === 0)
+            varint();
+        else if (wire === 1)
+            i += 8;
+        else if (wire === 5)
+            i += 4;
+        else if (wire === 2) {
+            const length = varint();
+            fields.push([field, buf.subarray(i, i + length)]);
+            i += length;
+        }
+        else {
+            throw new Error('The package header is not readable');
+        }
+    }
+    return fields;
+}
+/**
+ * Writes a zip's files under `dir`: stored or deflated entries, which
+ * is what a CRX holds. A name that would land outside `dir` is refused.
+ */
+export async function unzipTo(zip, dir) {
+    // The end of central directory record, last in the file but for a
+    // comment of up to 64KB.
+    let end = -1;
+    for (let i = zip.length - 22; i >= Math.max(0, zip.length - 22 - 65_535); i--) {
+        if (zip.readUInt32LE(i) === 0x06054b50) {
+            end = i;
+            break;
+        }
+    }
+    if (end < 0)
+        throw new Error('The package\'s zip has no central directory');
+    const count = zip.readUInt16LE(end + 10);
+    let at = zip.readUInt32LE(end + 16);
+    const root = path.resolve(dir);
+    await mkdir(root, { recursive: true });
+    for (let n = 0; n < count; n++) {
+        if (zip.readUInt32LE(at) !== 0x02014b50)
+            throw new Error('The package\'s zip directory is damaged');
+        const method = zip.readUInt16LE(at + 10);
+        const size = zip.readUInt32LE(at + 20);
+        const nameLength = zip.readUInt16LE(at + 28);
+        const extraLength = zip.readUInt16LE(at + 30);
+        const commentLength = zip.readUInt16LE(at + 32);
+        const local = zip.readUInt32LE(at + 42);
+        const name = zip.toString('utf8', at + 46, at + 46 + nameLength);
+        at += 46 + nameLength + extraLength + commentLength;
+        const target = path.resolve(root, name);
+        if (target !== root && !target.startsWith(root + path.sep))
+            throw new Error(`The package holds a file outside its folder: ${name}`);
+        if (name.endsWith('/')) {
+            await mkdir(target, { recursive: true });
+            continue;
+        }
+        const start = local + 30 + zip.readUInt16LE(local + 26) + zip.readUInt16LE(local + 28);
+        const data = zip.subarray(start, start + size);
+        let body;
+        if (method === 0)
+            body = data;
+        else if (method === 8)
+            body = inflateRawSync(data);
+        else
+            throw new Error(`The package's file ${name} is compressed in a way sm-dev can't read (method ${method})`);
+        await mkdir(path.dirname(target), { recursive: true });
+        await writeFile(target, body);
+    }
+}

package/lib/harness/managers.d.ts

@@ -0,0 +1,24 @@
+import type { ManagerName } from './json-output.js';
+export type { ManagerName };
+export interface ManagerInfo {
+    name: ManagerName;
+    /** `sm`, `tm`, `vm`: what `--manager` takes, besides the full name. */
+    short: string;
+    label: string;
+    /**
+     * Its Chrome Web Store id, which `manager fetch` downloads by. The
+     * same as in tests/fixtures/managers.mjs, which can't import this.
+     * SourceMonkey comes with sm-dev instead.
+     */
+    storeId?: string;
+}
+export declare const MANAGERS: Record<ManagerName, ManagerInfo>;
+export declare const MANAGER_NAMES: ManagerName[];
+/** The default, when `--manager` isn't given. */
+export declare const DEFAULT_MANAGER: ManagerName;
+/** The manager `value` names (`tm`, `Tampermonkey`, any case), or null. */
+export declare function managerNamed(value: string): ManagerName | null;
+/** What `--manager` says of a value it doesn't take. */
+export declare function managerMistake(value: string): string;
+/** `--manager tm`, the flag that picks it, for a command to show. */
+export declare function managerFlag(name: ManagerName): string;

package/lib/harness/managers.js

@@ -0,0 +1,27 @@
+// The userscript managers sm-dev knows: SourceMonkey, Tampermonkey (TM)
+// and Violentmonkey (VM). `--manager` picks one for the whole line,
+// and each command says which it works with (command-list.ts).
+//
+// It imports only json-output.ts, which imports nothing, so
+// command-list.ts can load it for help.
+export const MANAGERS = {
+    sourcemonkey: { name: 'sourcemonkey', short: 'sm', label: 'SourceMonkey' },
+    tampermonkey: { name: 'tampermonkey', short: 'tm', label: 'Tampermonkey', storeId: 'dhdgffkkebhmkfjojejmpbldmpobfkfo' },
+    violentmonkey: { name: 'violentmonkey', short: 'vm', label: 'Violentmonkey', storeId: 'jinjaccalgkegednnccohejagnlnfdag' },
+};
+export const MANAGER_NAMES = Object.keys(MANAGERS);
+/** The default, when `--manager` isn't given. */
+export const DEFAULT_MANAGER = 'sourcemonkey';
+/** The manager `value` names (`tm`, `Tampermonkey`, any case), or null. */
+export function managerNamed(value) {
+    const v = value.trim().toLowerCase();
+    return MANAGER_NAMES.find((name) => name === v || MANAGERS[name].short === v) ?? null;
+}
+/** What `--manager` says of a value it doesn't take. */
+export function managerMistake(value) {
+    return `There is no manager ${value}. The --manager flag takes sm, tm or vm, or sourcemonkey, tampermonkey or violentmonkey`;
+}
+/** `--manager tm`, the flag that picks it, for a command to show. */
+export function managerFlag(name) {
+    return `--manager ${MANAGERS[name].short}`;
+}

package/lib/harness/output-style.d.ts

@@ -0,0 +1,125 @@
+/**
+ * Starts a line's markup: a control character and a private-use one, a
+ * pair page text won't hold by chance (icon fonts use U+E000 alone),
+ * and invisible if it ever leaks.
+ */
+export declare const MARKUP = "\u0002\uE000";
+/** What a piece of output is. Each has one style, in STYLES. */
+export type Role = 'prompt' | 'typed' | 'cmd' | 'code' | 'err' | 'warn' | 'h' | 'sel' | 'id' | 'dim' | 'url' | 'name';
+/**
+ * A role's SGR codes, on and off, for text painted elsewhere (help's
+ * code and headings). Off undoes only that role's looks, so it can
+ * nest inside another (code in a bold heading), as long as the two
+ * share no look: were `cmd` bold too, its off would end the heading's
+ * bold.
+ */
+export declare function styleCodes(roleName: Role): [string, string];
+/**
+ * Whether a code span keeps its backticks in plain text (help, and sm-dev's messages): one of several
+ * words (`--lang zh-CN`), a lowercase word that reads as English (a
+ * command, `all`), or punctuation (`{{`). A flag, ID, number, path or
+ * name reads as code without them.
+ */
+export declare function keepsBackticks(span: string): boolean;
+/** Starts sm-dev's own messages. */
+export declare const SM_DEV_PREFIX = "[sm-dev] ";
+/** Starts the client's line for a command given wrongly. */
+export declare const ERROR_PREFIX = "ERROR: ";
+/** Starts a run's line for a command given wrongly (exit code 2). */
+export declare const MISTAKE_PREFIX = "[sm-dev] ERROR: ";
+/** Starts a warning about the browser. */
+export declare const WARNING_PREFIX = "Warning: ";
+/** Starts a script's warning from preparing it (`warningLines`). */
+export declare const WARNING_LEVEL = "warn: ";
+/** Starts a script's warning from preparing it, as a run prints it. */
+export declare const SCRIPT_WARNING_PREFIX = "[sm-dev] warn: ";
+/** Starts a failed `wait`. */
+export declare const WAIT_FAILED = "[sm-dev] Wait failed: ";
+/** Starts `validate`'s line for an invalid script, and a run's. */
+export declare const INVALID_PREFIX = "INVALID  ";
+/**
+ * Text in backticks, for a message: a command, flag or the user's input
+ * (`` Use `tabs` ``). Shown as a command in color, and plain with its
+ * backticks where they help it read (see keepsBackticks). Text holding a
+ * backtick itself (a selector, say) goes in double ones, as Markdown
+ * does: ``` `` a`b `` ```.
+ */
+export declare function quoted(text: unknown): string;
+/** Text escaped for markup. */
+export declare function escapeMarkup(text: string): string;
+/** A table cell: its text, and the same as markup. Columns line up by the text. */
+export interface Cell {
+    plain: string;
+    styled: string;
+}
+/** A cell of plain text. */
+export declare const textCell: (text: string) => Cell;
+/** A cell in a role: `roleCell(url, 'https://x/')`. */
+export declare const roleCell: (style: (text: string) => string, text: string) => Cell;
+/**
+ * Rows as lines: columns two spaces apart, each as wide as its widest
+ * cell. The last column isn't padded, so no line ends in spaces. As
+ * markup, or with `markup: false`, as plain text.
+ */
+export declare function tableLines(rows: Cell[][], { markup }?: {
+    markup?: boolean;
+}): string[];
+/** Text as a role: `role('id', 'T1')`. Text that is markup already stays markup. */
+export declare function role(name: Role, text: string | number): string;
+export declare const id: (text: string | number) => string;
+export declare const name: (text: string) => string;
+export declare const url: (text: string) => string;
+export declare const cmd: (text: string) => string;
+export declare const code: (text: string) => string;
+export declare const heading: (text: string) => string;
+export declare const selected: (text: string) => string;
+export declare const dim: (text: string) => string;
+/** A message we wrote, as an error: its `backticked` spans are commands (see `message`). */
+export declare const err: (text: string) => string;
+/** A message we wrote, as a warning. */
+export declare const warn: (text: string) => string;
+/** Text we didn't write (a script's console line, a page's error), as an error, taken as it is. */
+export declare const asError: (text: string) => string;
+/** Text we didn't write, as a warning. */
+export declare const asWarning: (text: string) => string;
+/**
+ * A line of markup, from a template: what is put in is escaped, unless
+ * it is markup already (from `ui` or a role helper).
+ */
+export declare function ui(strings: TemplateStringsArray, ...values: unknown[]): string;
+/**
+ * A message we wrote, as markup: each `backticked` span is a command
+ * (`Use `tabs` to list them`), shown in color, or with its backticks
+ * where they help it read (see keepsBackticks). Text that is markup
+ * already stays as it is.
+ */
+export declare function message(text: string): string;
+/**
+ * A line as shown: painted for a terminal with color, or plain. Plain
+ * text that is not one of sm-dev's messages is returned as it is.
+ */
+export declare function paintOutput(line: string, color: boolean): string;
+/**
+ * Output that may span lines (a stack trace in one error) as lines that
+ * each stand alone, for a reader that paints line by line. A role open
+ * at a line break is closed there and opened again on the next line, so
+ * every line is markup of its own; without that, the lines after the
+ * first would show their tags and escapes as text.
+ * - Plain text before the markup stays as it is at the start of its
+ *   line, where readers look for a prefix (`[sm-dev] `).
+ * - Plain text with no markup is just split: each line is read on its
+ *   own, by its own prefix.
+ */
+export declare function markupLines(line: string): string[];
+/** A line as plain text: for the log file, and for clients that didn't ask for markup. */
+export declare function plainLine(line: string): string;
+/**
+ * Whether output to `stream` is painted: at a terminal that has color.
+ * FORCE_COLOR asks for it anyway (or, as 0 or false, not), as Node
+ * reads it; NO_COLOR is honored by hasColors.
+ */
+export declare function streamHasColor(stream: NodeJS.WriteStream): boolean;
+/** A command echoed before its output, as the prompt shows it: `arg > status`. */
+export declare function echoLine(mark: string, text: string): string;
+/** Text painted as one role, for a terminal with color: the prompt and typed commands. */
+export declare function paintAs(roleName: Role, text: string): string;