roxxie-proxy-u-prod 0.1.0

package/source/packages/scramjet/packages/core/src/fetch/index.ts

@@ -0,0 +1,198 @@
+import {
+	BareCompatibleClient,
+	BareResponse,
+	ProxyTransport,
+	BareRequestInit,
+} from "@mercuryworkshop/proxy-transports";
+
+import { type URLMeta } from "@rewriters/url";
+import { ScramjetHeaders } from "@/shared/headers";
+import { HtmlRewriterHooks, ScramjetContext } from "@/shared";
+import { Tap, TapInstance } from "@/Tap";
+import { doHandleFetch } from "./fetch";
+import { _URL, _Map } from "@/shared/snapshot";
+
+export interface ScramjetFetchRequest {
+	rawUrl: URL;
+	rawReferrer: string | null;
+	// use parsed.destination instead
+	rawDestination: RequestDestination;
+	mode: RequestMode;
+	referrer: string;
+	method: string;
+	body: BodyType | null;
+	cache: RequestCache;
+
+	initialHeaders: ScramjetHeaders;
+
+	rawClientUrl?: URL;
+
+	/** The service worker FetchEvent.clientId that originated this request. */
+	clientId: string;
+}
+
+export interface ScramjetFetchParsed {
+	url: _URL;
+	clientUrl?: _URL;
+	referrerSourceUrl?: _URL | null;
+	hadExtraParams: boolean;
+	crossSiteRedirect: boolean;
+
+	// track the worst case Sec-Fetch-Site classification through redirects
+	fetchSiteState?: "same-origin" | "same-site" | "cross-site";
+
+	// origin of the page that initialized the request
+	// specifically for tracking Sec-Fetch-Site, don't use for anything else, it will diverge from clientUrl in some cases
+	fetchInitiatorOrigin?: string;
+
+	// was the request made with credentials=include?
+	fetchCredentialsInclude?: boolean;
+
+	// tracks RequestInit.mode if set
+	fetchMode?: RequestMode;
+
+	// was this request made by an iframe? (scramjet's definition of an iframe, not the browser's)
+	isIframe?: boolean;
+
+	// request.destination, but is overridden by $dest
+	destination: RequestDestination;
+
+	meta: URLMeta;
+	isModule: boolean;
+	isFakeDataURL: boolean;
+	referrerPolicy?: string;
+	trackedClient?: ScramjetFetchTrackedClient;
+}
+
+export interface ScramjetFetchResponse {
+	body: BodyType;
+	headers: ScramjetHeaders;
+	status: number;
+	statusText: string;
+}
+
+export type CookieSyncEntry = {
+	url: URL;
+	cookie: string;
+};
+
+export type CookieSyncOptions = {
+	clear?: boolean;
+	destination?: RequestDestination;
+};
+
+export type FetchHandlerInit = {
+	transport: ProxyTransport;
+	context: ScramjetContext;
+	crossOriginIsolated?: boolean;
+
+	sendSetCookie: (
+		cookies: CookieSyncEntry[],
+		options?: CookieSyncOptions
+	) => Promise<void>;
+	fetchDataUrl(dataUrl: string): Promise<BareResponse>;
+	fetchBlobUrl(blobUrl: string): Promise<BareResponse>;
+};
+
+export type TrackedHistoryState = {
+	url: string;
+	refererPolicy?: string;
+};
+export class ScramjetFetchTrackedClient {
+	history: TrackedHistoryState[] = [];
+	constructor(public clientId: string) {}
+}
+
+// eslint-disable-next-line scramjet-core/no-globals
+export class ScramjetFetchHandler extends EventTarget {
+	public client: BareCompatibleClient;
+	public crossOriginIsolated: boolean = false;
+	public context: ScramjetContext;
+
+	public trackedClients: Map<string, ScramjetFetchTrackedClient> = new _Map<
+		string,
+		ScramjetFetchTrackedClient
+	>();
+
+	public hooks: {
+		rewriter: {
+			html: TapInstance<HtmlRewriterHooks>;
+		};
+		fetch: TapInstance<FetchHooks>;
+	};
+
+	public fetchDataUrl: (dataUrl: string) => Promise<Response>;
+	public fetchBlobUrl: (blobUrl: string) => Promise<Response>;
+	public sendSetCookie: (
+		cookies: CookieSyncEntry[],
+		options?: CookieSyncOptions
+	) => Promise<void>;
+
+	constructor(init: FetchHandlerInit) {
+		super();
+		this.client = new BareCompatibleClient(init.transport);
+		this.context = init.context;
+		this.crossOriginIsolated = init.crossOriginIsolated || false;
+		this.sendSetCookie = init.sendSetCookie;
+		this.fetchDataUrl = init.fetchDataUrl;
+		this.fetchBlobUrl = init.fetchBlobUrl;
+		this.hooks = {
+			rewriter: {
+				html: Tap.create<HtmlRewriterHooks>(),
+			},
+			fetch: Tap.create<FetchHooks>(),
+		};
+		this.context.hooks = {
+			rewriter: this.hooks.rewriter,
+		};
+	}
+
+	async handleFetch(
+		request: ScramjetFetchRequest
+	): Promise<ScramjetFetchResponse> {
+		return doHandleFetch(this, request);
+	}
+}
+export type FetchHooks = {
+	intercept: {
+		context: {
+			request: ScramjetFetchRequest;
+			parsed: ScramjetFetchParsed;
+		};
+		props: {
+			response?: ScramjetFetchResponse;
+		};
+	};
+	request: {
+		context: {
+			request: ScramjetFetchRequest;
+			parsed: ScramjetFetchParsed;
+			client: BareCompatibleClient;
+		};
+		props: {
+			init: BareRequestInit;
+			url: URL;
+			earlyResponse?: BareResponse;
+		};
+	};
+	preresponse: {
+		context: {
+			request: ScramjetFetchRequest;
+			parsed: ScramjetFetchParsed;
+		};
+		props: {
+			response: BareResponse;
+		};
+	};
+	response: {
+		context: {
+			request: ScramjetFetchRequest;
+			parsed: ScramjetFetchParsed;
+		};
+		props: {
+			response: ScramjetFetchResponse;
+		};
+	};
+};
+
+export type BodyType = string | ArrayBuffer | Blob | ReadableStream<any>;

package/source/packages/scramjet/packages/core/src/fetch/parse.ts

@@ -0,0 +1,149 @@
+import { Object_entries, Object_keys, _URL, Error } from "@/shared/snapshot";
+import { unrewriteUrl, URLMeta } from "@rewriters/url";
+import {
+	ScramjetFetchHandler,
+	ScramjetFetchParsed,
+	ScramjetFetchRequest,
+	ScramjetFetchTrackedClient,
+} from ".";
+
+export const QP = {
+	referrerPolicy: "$rfp",
+	referrerSource: "$rfs",
+	isModule: "$module",
+	topFrame: "$tf",
+	parentFrame: "$pf",
+	isIframe: "$iframe",
+	mode: "$mode",
+	credentials: "$cred",
+	destination: "$dest",
+	initiatorOrigin: "$io",
+	fetchSite: "$fs",
+	crossSiteRedirect: "$csr",
+	fakeDataURL: "$fakedataurl",
+} as const;
+
+export type QueryParamKey = keyof typeof QP;
+
+export type QueryParams = Partial<Record<QueryParamKey, string>>;
+
+const QP_INVERSE: Record<string, QueryParamKey> = (() => {
+	const inv: Record<string, QueryParamKey> = {};
+	for (const key of Object_keys(QP) as QueryParamKey[]) {
+		inv[QP[key]] = key;
+	}
+	return inv;
+})();
+
+export function parseQueryParams(searchParams: URLSearchParams): {
+	params: QueryParams;
+	extras: Record<string, string>;
+} {
+	const params: QueryParams = {};
+	const extras: Record<string, string> = {};
+	for (const [key, value] of [...searchParams.entries()]) {
+		const logical = QP_INVERSE[key];
+		if (logical) {
+			params[logical] = value;
+		} else {
+			dbg.warn(
+				`extraneous query parameter ${key}=${value}. Assuming <form> element`
+			);
+			extras[key] = value;
+		}
+	}
+	return { params, extras };
+}
+
+export function parseRequest(
+	request: ScramjetFetchRequest,
+	handler: ScramjetFetchHandler
+): ScramjetFetchParsed {
+	const strippedUrl = new _URL(request.rawUrl.href);
+	const { params, extras } = parseQueryParams(request.rawUrl.searchParams);
+	strippedUrl.search = "";
+
+	const hadExtraParams = Object_keys(extras).length > 0;
+
+	if (!_URL.canParse(unrewriteUrl(strippedUrl, handler.context))) {
+		throw new Error(`unable to parse rewritten url: ${strippedUrl.href}`);
+	}
+	const url = new _URL(unrewriteUrl(strippedUrl, handler.context));
+
+	if (url.origin === new _URL(request.rawUrl).origin) {
+		// uh oh!
+		throw new Error(
+			"attempted to fetch from same origin - this means the site has obtained a reference to the real origin, aborting"
+		);
+	}
+
+	for (const [key, value] of Object_entries(extras)) {
+		url.searchParams.set(key, value);
+	}
+
+	const clientId = request.clientId;
+	let trackedClient: ScramjetFetchTrackedClient | undefined;
+	if (clientId) {
+		trackedClient = handler.trackedClients.get(clientId);
+		if (!trackedClient) {
+			trackedClient = new ScramjetFetchTrackedClient(clientId);
+			handler.trackedClients.set(clientId, trackedClient);
+		}
+	}
+
+	const referrerSourceUrl =
+		params.referrerSource === undefined
+			? undefined
+			: params.referrerSource
+				? new _URL(params.referrerSource)
+				: null;
+
+	const fetchSiteState =
+		params.fetchSite === "same-origin" ||
+		params.fetchSite === "same-site" ||
+		params.fetchSite === "cross-site"
+			? params.fetchSite
+			: undefined;
+
+	const fetchMode = (
+		["cors", "no-cors", "same-origin", "navigate"] as const
+	).find((mode) => mode === params.mode);
+	const destination =
+		(params.destination as RequestDestination | undefined) ||
+		request.rawDestination;
+
+	const meta: URLMeta = {
+		origin: url,
+		base: url,
+		topFrameName: params.topFrame,
+		parentFrameName: params.parentFrame,
+		referrerPolicy: params.referrerPolicy,
+	};
+
+	const parsed: ScramjetFetchParsed = {
+		meta,
+		url,
+		isModule: params.isModule === "module",
+		referrerPolicy: params.referrerPolicy,
+		referrerSourceUrl,
+		trackedClient,
+		hadExtraParams,
+		crossSiteRedirect: params.crossSiteRedirect === "1",
+		fetchSiteState,
+		fetchInitiatorOrigin: params.initiatorOrigin || undefined,
+		// TODO: should really just be a boolean
+		fetchCredentialsInclude: params.credentials === "include",
+		fetchMode,
+		destination,
+		isIframe: params.isIframe === "1",
+		isFakeDataURL: params.fakeDataURL === "1",
+	};
+
+	if (request.rawClientUrl) {
+		parsed.clientUrl = new _URL(
+			unrewriteUrl(request.rawClientUrl, handler.context)
+		);
+	}
+
+	return parsed;
+}

package/source/packages/scramjet/packages/core/src/fetch/util.ts

@@ -0,0 +1,81 @@
+import { isHtmlMimeType, ScramjetHeaders } from "@/shared";
+import { BareResponse } from "@mercuryworkshop/proxy-transports";
+import { ScramjetFetchParsed } from ".";
+
+export function normalizeContentType(
+	parsed: ScramjetFetchParsed,
+	headers: ScramjetHeaders
+) {
+	if (!isDocument(parsed)) return;
+
+	const ct = headers.get("content-type");
+	if (!ct) return;
+	if (!isHtmlMimeType(ct)) return;
+
+	headers.set("content-type", "text/html; charset=utf-8");
+}
+
+export function isRedirect(response: BareResponse) {
+	return response.status >= 300 && response.status < 400;
+}
+
+export function isDocument(parsed: ScramjetFetchParsed) {
+	return parsed.destination === "document" || parsed.destination === "iframe";
+}
+
+export function createReferrerString(
+	clientUrl: URL,
+	resource: URL,
+	policy: string | null
+): string {
+	policy ||= "strict-origin-when-cross-origin";
+	const originIsHttps = clientUrl.protocol === "https:";
+	const destIsHttps = resource.protocol === "https:";
+
+	const isPotentialDowngrade = originIsHttps && !destIsHttps;
+
+	const isSameOrigin =
+		clientUrl.protocol === resource.protocol &&
+		clientUrl.host === resource.host;
+
+	const referrerOrigin = clientUrl.origin;
+
+	const referrerUrl = new URL(clientUrl.href);
+	referrerUrl.hash = "";
+	const referrerUrlString = referrerUrl.href;
+
+	switch (policy) {
+		case "no-referrer":
+			return "";
+
+		case "no-referrer-when-downgrade":
+			if (isPotentialDowngrade) return "";
+			return referrerUrlString;
+
+		case "same-origin":
+			if (isSameOrigin) return referrerUrlString;
+			return "";
+
+		case "origin":
+			return referrerOrigin === "null" ? "" : referrerOrigin + "/";
+
+		case "strict-origin":
+			if (isPotentialDowngrade) return "";
+			return referrerOrigin === "null" ? "" : referrerOrigin + "/";
+
+		case "origin-when-cross-origin":
+			if (isSameOrigin) return referrerUrlString;
+			return referrerOrigin === "null" ? "" : referrerOrigin + "/";
+
+		case "strict-origin-when-cross-origin":
+			if (isSameOrigin) return referrerUrlString;
+			if (isPotentialDowngrade) return "";
+			return referrerOrigin === "null" ? "" : referrerOrigin + "/";
+
+		case "unsafe-url":
+			return referrerUrlString;
+
+		default:
+			return "";
+	}
+}

package/source/packages/scramjet/packages/core/src/global.d.ts

@@ -0,0 +1,32 @@
+/// <reference types="@rspack/core/module" />
+
+declare global {
+	interface Window {
+		WASM: string;
+		REAL_WASM: Uint8Array;
+
+		/**
+		 * The scramjet client belonging to a window.
+		 */
+		[import("./symbols").SCRAMJETCLIENT]: import("./client").ScramjetClient;
+	}
+
+	interface Document {
+		/**
+		 * Should be the same as window.
+		 */
+		[import("./symbols").SCRAMJETCLIENT]: import("./client").ScramjetClient;
+	}
+}
+
+declare const dbg: {
+	log: (message: string, ...args: any[]) => void;
+	warn: (message: string, ...args: any[]) => void;
+	error: (message: string, ...args: any[]) => void;
+	debug: (message: string, ...args: any[]) => void;
+	time: (meta: URLMeta, before: number, type: string) => void;
+};
+
+// eslint-disable-next-line scramjet-core/no-globals
+declare type GlobalThis = typeof globalThis;
+declare type Self = Window & GlobalThis;

package/source/packages/scramjet/packages/core/src/index.ts

@@ -0,0 +1,78 @@
+// NOTE: this is the entrypoint for scramjet.bundle.js
+// as such it exports everything in scramjet
+// the entry point for scramjet.all.js (what most sites wil use) is entry.ts
+
+import "./global.d";
+import { atob } from "@/shared/snapshot";
+import { setWasm } from "@rewriters/wasm";
+import { ScramjetVersionInfo, ScramjetConfig } from "./types";
+
+declare const VERSION: string;
+declare const COMMITHASH: string;
+declare const BUILDDATE: string;
+export const versionInfo: ScramjetVersionInfo = {
+	version: VERSION,
+	build: COMMITHASH,
+	date: BUILDDATE,
+};
+
+export const defaultConfig: ScramjetConfig = {
+	globals: {
+		wrapfn: "$scramjet$wrap",
+		wrappropertybase: "$scramjet__",
+		wrappropertyfn: "$scramjet$prop",
+		cleanrestfn: "$scramjet$clean",
+		importfn: "$scramjet$import",
+		rewritefn: "$scramjet$rewrite",
+		metafn: "$scramjet$meta",
+		wrappostmessagefn: "$scramjet$wrappostmessage",
+		pushsourcemapfn: "$scramjet$pushsourcemap",
+		trysetfn: "$scramjet$tryset",
+		templocid: "$scramjet$temploc",
+		tempunusedid: "$scramjet$tempunused",
+	},
+	flags: {
+		syncxhr: false,
+		disableComputedWrap: false,
+		rewriterLogs: false,
+		captureErrors: false,
+		cleanErrors: false,
+		scramitize: false,
+		sourcemaps: true,
+		destructureRewrites: true,
+		allowInvalidJs: true,
+		debugTrampolines: false,
+		allowFailedIntercepts: false,
+		encapsulateWorkers: true,
+		debugSourceURL: false,
+	},
+	siteFlags: {},
+	maskedfiles: [],
+};
+
+export const defaultConfigDev: ScramjetConfig = {
+	...defaultConfig,
+	flags: {
+		...defaultConfig.flags,
+		rewriterLogs: false,
+		captureErrors: true,
+		cleanErrors: false,
+		debugTrampolines: true,
+		debugSourceURL: true,
+		allowInvalidJs: false,
+	},
+};
+
+declare const REWRITERWASM: string | undefined;
+// bundled build will have the wasm binary inlined as a base64 string
+if (REWRITERWASM) {
+	setWasm(Uint8Array.from(atob(REWRITERWASM), (c) => c.charCodeAt(0)));
+}
+
+export * from "./symbols";
+export * from "./types";
+export * from "./Tap";
+export * from "./shared";
+export * from "./fetch";
+export { BareResponse } from "@mercuryworkshop/proxy-transports";
+export * from "./client";

package/source/packages/scramjet/packages/core/src/log.ts

@@ -0,0 +1,117 @@
+// import { flagEnabled } from "@/shared";
+import type { URLMeta } from "@rewriters/url";
+import { Error, Math_min, Performance_now } from "@/shared/snapshot";
+
+const logfuncs = {
+	// eslint-disable-next-line scramjet-core/no-globals
+	log: console.log,
+	// eslint-disable-next-line scramjet-core/no-globals
+	warn: console.warn,
+	// eslint-disable-next-line scramjet-core/no-globals
+	error: console.error,
+	// eslint-disable-next-line scramjet-core/no-globals
+	debug: console.debug,
+	// eslint-disable-next-line scramjet-core/no-globals
+	info: console.info,
+};
+
+export default {
+	fmt: function (severity: string, message: string, ...args: any[]) {
+		const old = Error.prepareStackTrace;
+
+		Error.prepareStackTrace = (_, stack) => {
+			stack.shift(); // stack();
+			stack.shift(); // fmt();
+			stack.shift();
+
+			let fmt = "";
+			for (let i = 1; i < Math_min(2, stack.length); i++) {
+				if (stack[i].getFunctionName()) {
+					// const f = stack[i].getThis()?.constructor?.name;
+					// if (f) fmt += `${f}.`
+					fmt += `${stack[i].getFunctionName()} -> ` + fmt;
+				}
+			}
+			fmt += stack[0].getFunctionName() || "Anonymous";
+
+			return fmt;
+		};
+
+		const fmt = (function stack() {
+			try {
+				throw new Error();
+			} catch (e) {
+				return e.stack;
+			}
+		})();
+
+		Error.prepareStackTrace = old;
+
+		this.print(severity, fmt, message, ...args);
+	},
+	print(severity: string, tag: string, message: string, ...args: any[]) {
+		const fn = logfuncs[severity] || logfuncs.log;
+		const bg = {
+			log: "#000",
+			warn: "#f80",
+			error: "#f00",
+			debug: "transparent",
+		}[severity];
+		const fg = {
+			log: "#fff",
+			warn: "#fff",
+			error: "#fff",
+			debug: "gray",
+		}[severity];
+		const padding = {
+			log: 2,
+			warn: 4,
+			error: 4,
+			debug: 0,
+		}[severity];
+
+		fn(
+			`%c${tag}%c ${message}`,
+			`
+  	background-color: ${bg};
+  	color: ${fg};
+  	padding: ${padding}px;
+  	font-weight: bold;
+  	font-family: monospace;
+  	font-size: 0.9em;
+  `,
+			`${severity === "debug" ? "color: gray" : ""}`,
+			...args
+		);
+	},
+	log: function (message: string, ...args: any[]) {
+		this.fmt("log", message, ...args);
+	},
+	warn: function (message: string, ...args: any[]) {
+		this.fmt("warn", message, ...args);
+	},
+	error: function (message: string, ...args: any[]) {
+		this.fmt("error", message, ...args);
+	},
+	debug: function (message: string, ...args: any[]) {
+		this.fmt("debug", message, ...args);
+	},
+	time(meta: URLMeta, before: number, type: string) {
+		const after = Performance_now();
+		const duration = after - before;
+
+		let timespan: string;
+		if (duration < 1) {
+			timespan = "BLAZINGLY FAST";
+		} else if (duration < 500) {
+			timespan = "decent speed";
+		} else {
+			timespan = "really slow";
+		}
+		this.print(
+			"debug",
+			"[time]",
+			`${type} was ${timespan} (${duration.toFixed(2)}ms)`
+		);
+	},
+};