universal-parcel-scraper 0.191.0-main.1349 → 0.191.0

package/dist/carriers/royal-mail/adapter.d.ts

@@ -4,7 +4,9 @@ import { type StepRecorder } from '../../core/telemetry/index.js';
 import { TrawlClient } from '../../core/transport/index.js';
 export { normalizeRoyalMailNumber, parseRoyalMailTrackingHtml, parseRoyalMailTrackingResponse, royalMailEventsApiUrl, royalMailSummaryApiUrl, royalMailTrackingUrl } from './parser.js';
 export interface RoyalMailTrackerOptions {
-    /** Fresh local Chromium; preferred when supplied. */
+    /** Private Playwright WebSocket service; preferred when supplied. Keep its capability URL outside logs and Git. */
+    browserWsEndpoint?: string | null;
+    /** Fresh local Chromium, used when no private browser endpoint is supplied. */
     executablePath?: string | null;
     /** An `http://` or `socks5://` proxy for the local Chromium's traffic. */
     browserProxy?: string | null;

package/dist/carriers/royal-mail/adapter.js

@@ -20,6 +20,7 @@ export class RoyalMailTracker {
     #fullHistory;
     #executablePath;
     #browserProxy;
+    #browserWsEndpoint;
     constructor(options = {}) {
         if (options.timeoutMs !== undefined && (!Number.isFinite(options.timeoutMs) || options.timeoutMs <= 0)) {
             throw new TypeError('Royal Mail timeout must be positive');
@@ -32,6 +33,7 @@ export class RoyalMailTracker {
         this.#fullHistory = options.fullHistory ?? false;
         this.#executablePath = options.executablePath ?? null;
         this.#browserProxy = options.browserProxy?.trim() || null;
+        this.#browserWsEndpoint = options.browserWsEndpoint?.trim() || null;
     }
     /** The injected browser service, or one built from the configured URL. */
     #browserService() {
@@ -41,9 +43,9 @@ export class RoyalMailTracker {
     }
     async fetch(trackingNumber, context = {}) {
         const number = normalizeRoyalMailNumber(trackingNumber);
-        if (this.#executablePath) {
+        if (this.#browserWsEndpoint || this.#executablePath) {
             return runSteps({ carrier: 'royal-mail', budgetMs: context.budgetMs ?? this.timeoutMs,
-                signal: context.signal, recorder: this.#recorder }, [{ id: 'browser', run: ({ remainingMs, signal }) => fetchRoyalMailInBrowser(number, this.#executablePath, signal, Math.min(60_000, remainingMs), this.#fullHistory, this.#browserProxy) }]);
+                signal: context.signal, recorder: this.#recorder }, [{ id: 'browser', run: ({ remainingMs, signal }) => fetchRoyalMailInBrowser(number, this.#executablePath ?? '', signal, Math.min(60_000, remainingMs), this.#fullHistory, this.#browserProxy, this.#browserWsEndpoint) }]);
         }
         // The legacy service remains available through explicit tracker instances.
         const trawl = this.#browserService();
@@ -170,20 +172,21 @@ export class RoyalMailTracker {
     }
 }
 export const adapter = environment => {
+    const endpoint = environment.env.ROYAL_MAIL_BROWSER_WS_ENDPOINT?.trim() || null;
     const tracker = new RoyalMailTracker({ executablePath: environment.browserExecutablePath,
-        browserProxy: environment.env.ROYAL_MAIL_BROWSER_PROXY, recorder: environment.recorder, fullHistory: true });
+        browserWsEndpoint: endpoint, browserProxy: environment.env.ROYAL_MAIL_BROWSER_PROXY, recorder: environment.recorder, fullHistory: true });
     return {
         id: 'royal-mail', recordsSteps: true, steps: ['browser'],
         track: (input, context) => {
             normalizeRoyalMailNumber(input.number);
-            if (!environment.browserExecutablePath)
-                throw new ChallengeError('Royal Mail', 'Royal Mail requires TRACKING_CHROMIUM_PATH');
+            if (!endpoint && !environment.browserExecutablePath)
+                throw new ChallengeError('Royal Mail', 'Royal Mail requires TRACKING_CHROMIUM_PATH or ROYAL_MAIL_BROWSER_WS_ENDPOINT');
             return tracker.fetch(input.number, context);
         },
         recognizeWithBrowser: (number, context) => recognizeFromBrowserLookup(() => {
             normalizeRoyalMailNumber(number);
-            if (!environment.browserExecutablePath)
-                throw new ChallengeError('Royal Mail', 'Royal Mail requires TRACKING_CHROMIUM_PATH');
+            if (!endpoint && !environment.browserExecutablePath)
+                throw new ChallengeError('Royal Mail', 'Royal Mail requires TRACKING_CHROMIUM_PATH or ROYAL_MAIL_BROWSER_WS_ENDPOINT');
             return tracker.fetch(number, context);
         }),
     };

package/dist/carriers/royal-mail/browser-service.d.ts

@@ -0,0 +1,3 @@
+import type { BrowserServer } from 'playwright-core';
+/** A private Playwright service. Its capability path and process environment never enter browser pages. */
+export declare function startRoyalMailBrowserService(env: Readonly<Record<string, string | undefined>>): Promise<BrowserServer>;

package/dist/carriers/royal-mail/browser-service.js

@@ -0,0 +1,19 @@
+import { loadChromium } from '../../core/transport/optional.js';
+/** A private Playwright service. Its capability path and process environment never enter browser pages. */
+export async function startRoyalMailBrowserService(env) {
+    const executablePath = env.TRACKING_CHROMIUM_PATH?.trim();
+    if (!executablePath)
+        throw new TypeError('Royal Mail browser service requires TRACKING_CHROMIUM_PATH');
+    const wsPath = env.ROYAL_MAIL_BROWSER_WS_PATH ?? '';
+    if (!/^\/[A-Za-z0-9_-]{32,128}$/.test(wsPath)) {
+        throw new TypeError('ROYAL_MAIL_BROWSER_WS_PATH must be an unguessable path of 32 to 128 URL-safe characters');
+    }
+    const rawPort = env.ROYAL_MAIL_BROWSER_PORT ?? '3001';
+    const port = /^\d{1,5}$/.test(rawPort) ? Number(rawPort) : 0;
+    if (port < 1 || port > 65535)
+        throw new TypeError('Invalid Royal Mail browser service port');
+    const chromium = await loadChromium('Royal Mail');
+    return chromium.launchServer({ executablePath, headless: true, host: '127.0.0.1', port, wsPath,
+        args: ['--disable-blink-features=AutomationControlled'], timeout: 20_000,
+        env: { PATH: env.PATH ?? '', HOME: env.HOME ?? '/tmp', LANG: 'en_US.UTF-8' } });
+}

package/dist/carriers/royal-mail/browser.d.ts

@@ -1,5 +1,6 @@
-import type { Page } from 'playwright-core';
+import type { Browser, Page } from 'playwright-core';
 import type { CarrierResult } from '../../core/result/index.js';
+export declare function royalMailBrowserUserAgent(browser: Browser, remote: boolean): Promise<string>;
 /** The destination the full-history reply adds to the summary, and nothing else from it. */
 export declare function royalMailDestination(summary: unknown): {
     summary?: {
@@ -8,4 +9,4 @@ export declare function royalMailDestination(summary: unknown): {
 };
 /** Observe the application's request; its issued session stays in the browser. */
 export declare function captureRoyalMailReply(page: Page, url: string, number: string, phase: 'summary' | 'events', signal: AbortSignal, timeoutMs: number, trigger: () => Promise<void>): Promise<Record<string, unknown>>;
-export declare function fetchRoyalMailInBrowser(number: string, executablePath: string, signal: AbortSignal, timeoutMs: number, fullHistory: boolean, proxy?: string | null): Promise<CarrierResult>;
+export declare function fetchRoyalMailInBrowser(number: string, executablePath: string, signal: AbortSignal, timeoutMs: number, fullHistory: boolean, proxy?: string | null, endpoint?: string | null): Promise<CarrierResult>;

package/dist/carriers/royal-mail/browser.js

@@ -1,6 +1,6 @@
 import { setTimeout as delay } from 'node:timers/promises';
 import { carrierErrorKind, ChallengeError, IndeterminateError, RateLimitedError, SchemaError, TransportError, UpstreamHttpError } from '../../core/errors/index.js';
-import { withLocalBrowser } from '../../core/transport/localBrowser.js';
+import { withLocalBrowser, withRemoteBrowser } from '../../core/transport/localBrowser.js';
 import { isRecord } from '../../core/types.js';
 import { parseRoyalMailTrackingResponse, royalMailEventsApiUrl, royalMailSummaryApiUrl, royalMailTrackingUrl } from './parser.js';
 const PROVIDER = 'Royal Mail';
@@ -8,6 +8,25 @@ const ORIGIN = 'https://www.royalmail.com';
 const MAX_BYTES = 2_000_000;
 // The page keeps this frame hidden off-screen until hCaptcha wants a person to solve it.
 const CAPTCHA_CHALLENGE = 'iframe[src*="hcaptcha.com"][src*="frame=challenge"]:visible';
+export async function royalMailBrowserUserAgent(browser, remote) {
+    const major = browser.version().split('.')[0];
+    if (remote) {
+        // Use the browser's OS, rather than the calling server's platform.
+        const cdp = await browser.newBrowserCDPSession();
+        try {
+            const version = await cdp.send('Browser.getVersion');
+            if (!/^Mozilla\/5\.0 [\x20-\x7E]{20,300}$/.test(version.userAgent))
+                throw new SchemaError(PROVIDER);
+            return version.userAgent.replace(/HeadlessChrome\/[\d.]+/, `Chrome/${major}.0.0.0`);
+        }
+        finally {
+            await cdp.detach();
+        }
+    }
+    const platform = process.platform === 'darwin' ? 'Macintosh; Intel Mac OS X 10_15_7'
+        : process.platform === 'win32' ? 'Windows NT 10.0; Win64; x64' : 'X11; Linux x86_64';
+    return `Mozilla/5.0 (${platform}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${major}.0.0.0 Safari/537.36`;
+}
 // The consent manager parks the address of a frame it blocks in `data-ta-src`.
 const CAPTCHA_WITHHELD = 'iframe[data-hcaptcha-widget-id][data-ta-src]';
 /** The destination the full-history reply adds to the summary, and nothing else from it. */
@@ -142,15 +161,14 @@ export async function captureRoyalMailReply(page, url, number, phase, signal, ti
         deadline.removeEventListener('abort', aborted);
     }
 }
-export function fetchRoyalMailInBrowser(number, executablePath, signal, timeoutMs, fullHistory, proxy = null) {
-    return withLocalBrowser({ provider: PROVIDER, executablePath, signal, timeoutMs, proxy,
+export function fetchRoyalMailInBrowser(number, executablePath, signal, timeoutMs, fullHistory, proxy = null, endpoint = null) {
+    const withBrowser = endpoint ? withRemoteBrowser : withLocalBrowser;
+    return withBrowser({ provider: PROVIDER, executablePath, signal, timeoutMs, proxy, endpoint: endpoint ?? '',
         args: ['--disable-blink-features=AutomationControlled'] }, async ({ browser, signal: session, remainingMs }) => {
         try {
-            const platform = process.platform === 'darwin' ? 'Macintosh; Intel Mac OS X 10_15_7'
-                : process.platform === 'win32' ? 'Windows NT 10.0; Win64; x64' : 'X11; Linux x86_64';
-            const major = browser.version().split('.')[0];
+            const userAgent = await royalMailBrowserUserAgent(browser, endpoint !== null);
             const context = await browser.newContext({ locale: 'en-GB', acceptDownloads: false, serviceWorkers: 'block',
-                userAgent: `Mozilla/5.0 (${platform}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${major}.0.0.0 Safari/537.36` });
+                userAgent });
             session.throwIfAborted();
             const page = await context.newPage();
             const timeout = () => Math.min(15_000, remainingMs());

package/dist/cli/index.js

@@ -5,6 +5,7 @@ import { InputRequiredError } from '../core/errors/index.js';
 import { createTracker, TrackingError } from '../facade/index.js';
 import { CARRIER_CATALOG } from '../generated/catalog.js';
 import { createTrackingServer } from '../server/index.js';
+import { startRoyalMailBrowserService } from '../carriers/royal-mail/browser-service.js';
 const help = `Universal Parcel Scraper
 
   parcel-scraper detect <number, link or text>
@@ -12,6 +13,7 @@ const help = `Universal Parcel Scraper
   parcel-scraper track <number> [--carrier <id>] [--postcode <value>] [--tracking-url <url>]
   parcel-scraper carriers
   parcel-scraper serve [--host <address>] [--port <port>]
+  parcel-scraper royal-mail-browser
 
 SCRAPER_PROVIDERS selects comma-separated providers; default: UPU.
 FLARESOLVERR_URL and TRACKING_CHROMIUM_PATH enable browser transports.
@@ -34,6 +36,15 @@ export async function main(argv = process.argv.slice(2), env = process.env) {
         console.log(help);
         return 0;
     }
+    if (command === 'royal-mail-browser') {
+        if (args.length)
+            throw new TypeError('Unexpected argument; configure the Royal Mail browser service through its environment');
+        const server = await startRoyalMailBrowserService(env);
+        console.error(JSON.stringify({ service: 'royal-mail-browser', state: 'ready' }));
+        for (const signal of ['SIGINT', 'SIGTERM'])
+            process.once(signal, () => { void server.close(); });
+        return 0;
+    }
     if (!['detect', 'recognize', 'track', 'carriers', 'serve'].includes(command))
         throw new TypeError('Unknown command; use --help');
     const values = {};

package/dist/core/transport/localBrowser.d.ts

@@ -1,5 +1,5 @@
 /**
- * The one local Chromium a process runs. A carrier outage can send a whole
+ * The one browser session a process runs. A carrier outage can send a whole
  * batch to the browser tier, so lookups take turns instead of each spawning a
  * browser: a lookup waits for the browser inside its own deadline and fails
  * with a budget error when that deadline passes first.
@@ -24,3 +24,7 @@ export interface LocalBrowserSession {
     remainingMs: () => number;
 }
 export declare function withLocalBrowser<T>(options: LocalBrowserOptions, run: (session: LocalBrowserSession) => Promise<T>): Promise<T>;
+/** Connects to a private Playwright browser; closing the client disposes its contexts, not the service. */
+export declare function withRemoteBrowser<T>(options: Pick<LocalBrowserOptions, 'provider' | 'signal' | 'timeoutMs'> & {
+    endpoint: string;
+}, run: (session: LocalBrowserSession) => Promise<T>): Promise<T>;

package/dist/core/transport/localBrowser.js

@@ -1,4 +1,4 @@
-import { BudgetExceededError, carrierErrorKind } from '../errors/index.js';
+import { BudgetExceededError, carrierErrorKind, TransportError } from '../errors/index.js';
 import { loadChromium } from './optional.js';
 /** Covers the rounding of the timeouts handed to Playwright and the granularity of timers. */
 const DEADLINE_SLACK_MS = 5;
@@ -17,6 +17,33 @@ function proxyServer(value) {
     return `${url.protocol}//${url.host}`;
 }
 export async function withLocalBrowser(options, run) {
+    return withBrowser(options, run);
+}
+/** Connects to a private Playwright browser; closing the client disposes its contexts, not the service. */
+export async function withRemoteBrowser(options, run) {
+    let url;
+    try {
+        url = new URL(options.endpoint);
+    }
+    catch {
+        throw new TypeError('Browser endpoint must be a WebSocket URL');
+    }
+    if (!['ws:', 'wss:'].includes(url.protocol) || !url.hostname || url.username || url.password || url.hash) {
+        throw new TypeError('Browser endpoint must be a WebSocket URL');
+    }
+    try {
+        return await withBrowser({ provider: options.provider, timeoutMs: options.timeoutMs, signal: options.signal, executablePath: '' }, run, url.href);
+    }
+    catch (error) {
+        if (options.signal?.aborted)
+            throw options.signal.reason;
+        if (carrierErrorKind(error))
+            throw error;
+        // Connection diagnostics can contain the endpoint's capability path.
+        throw new TransportError(options.provider, `${options.provider} browser service is unavailable`);
+    }
+}
+async function withBrowser(options, run, endpoint) {
     const { provider, timeoutMs } = options;
     if (!Number.isFinite(timeoutMs) || timeoutMs <= 0 || timeoutMs > 60_000) {
         throw new TypeError('Browser tracking timeout must be between 1 and 60000 ms');
@@ -69,9 +96,11 @@ export async function withLocalBrowser(options, run) {
         if (signal.aborted)
             throw stopped();
         // Do not pass the application environment (database or API secrets) to Chromium.
-        launching = chromium.launch({ executablePath: options.executablePath, headless: true, args: [...(options.args ?? [])],
-            timeout: Math.min(remainingMs(), 10_000), ...(proxy ? { proxy: { server: proxy } } : {}),
-            env: { PATH: process.env.PATH ?? '', HOME: process.env.HOME ?? '/tmp', LANG: 'en_US.UTF-8' } });
+        // The private service binds to loopback; its Host check must survive a tunnel's different address.
+        launching = endpoint ? chromium.connect(endpoint, { timeout: Math.min(remainingMs(), 10_000), headers: { Host: 'localhost' } })
+            : chromium.launch({ executablePath: options.executablePath, headless: true, args: [...(options.args ?? [])],
+                timeout: Math.min(remainingMs(), 10_000), ...(proxy ? { proxy: { server: proxy } } : {}),
+                env: { PATH: process.env.PATH ?? '', HOME: process.env.HOME ?? '/tmp', LANG: 'en_US.UTF-8' } });
         browser = await unlessStopped(launching);
         return await unlessStopped(run({ browser, signal, remainingMs }));
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "universal-parcel-scraper",
-  "version": "0.191.0-main.1349",
+  "version": "0.191.0",
   "description": "Self-hosted parcel tracking: carrier detection, dedicated scrapers and optional universal providers.",
   "type": "module",
   "license": "Apache-2.0",