xjs-common 7.0.0

package/dist/prcs/http/http-resolver-context.d.ts

@@ -0,0 +1,46 @@
+import { ClientMode } from "./http-resolver";
+import { ClientOption, IHttpClient, RequestOption } from "./i-http-client";
+export declare const s_clientMode: Record<string, ClientMode>;
+export declare class HttpResolverContext implements IHttpClient {
+    private readonly _cmv;
+    private _l;
+    private readonly _als;
+    private readonly _mode;
+    private readonly _ciphers;
+    private readonly _proxyConfig?;
+    private readonly _chHeaders;
+    private _cookies?;
+    constructor(_cmv: number, op?: ClientOption, _l?: {
+        log: (msg: any) => void;
+        warn: (msg: any) => void;
+    });
+    /**
+     * request GET to the url.
+     * @param url target url.
+     * @param op.headers http headers.
+     * @param op.ignoreQuery if true, query part in the `url` is ignored.
+     * @returns string encoded by utf-8 as response payload.
+     */
+    get(url: string, op?: RequestOption & {
+        outerRedirectCount?: number;
+    }): Promise<any>;
+    /**
+     * request POST to the url.
+     * @param url target url.
+     * @param payload request payload. if this is an object, it is treated as json.
+     * @param op.headers http headers.
+     * @param op.ignoreQuery if true, query part in the `url` is ignored.
+     * @returns string encoded by utf-8 as response payload.
+     */
+    post(url: string, payload: any, op?: RequestOption): Promise<any>;
+    private createProxyAgent;
+    private getIn;
+    private postIn;
+    private reqHttps;
+    private handleRedirect;
+    private createCiphers;
+    private setCookies;
+    private storeCookies;
+    private log;
+    private warn;
+}

package/dist/prcs/http/http-resolver-context.js

@@ -0,0 +1,287 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HttpResolverContext = exports.s_clientMode = void 0;
+const tls = __importStar(require("tls"));
+const zlib = __importStar(require("zlib"));
+const url_1 = require("url");
+const https_1 = require("https");
+const http_1 = require("http");
+const async_hooks_1 = require("async_hooks");
+const xjs_err_1 = require("../../obj/xjs-err");
+const u_http_1 = require("../../func/u-http");
+const u_array_1 = require("../../func/u-array");
+const http_method_1 = require("../../const/http-method");
+const u_type_1 = require("../../func/u-type");
+exports.s_clientMode = {
+    nodejs: { id: 0, cipherOrder: null },
+    chrome: { id: 1, cipherOrder: [2, 0, 1] },
+    firefox: { id: 2, cipherOrder: [2, 1, 0] }
+};
+const s_timeout = 1000 * 20;
+const s_errCode = 200;
+const s_redirectLimit = 5;
+const s_mode2headers = new Map([
+    [exports.s_clientMode.firefox, (cmv) => ({
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+            "Accept-Encoding": "gzip, deflate, br",
+            "Accept-Language": "en-US,en;q=0.5",
+            "Sec-Fetch-Dest": "document",
+            "Sec-Fetch-Mode": "navigate",
+            "Sec-Fetch-Site": "none",
+            "Sec-Fetch-User": "?1",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": `Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:${cmv}.0) Gecko/20100101 Firefox/${cmv}.0`
+        })],
+    [exports.s_clientMode.chrome, (cmv) => {
+            const uad = cmv < 130
+                ? `"Not/A)Brand";v="8", "Chromium";v="${cmv}", "Google Chrome";v="${cmv}"`
+                : `"Chromium";v="${cmv}", "Not:A-Brand";v="24", "Google Chrome";v="${cmv}"`;
+            const ch = {
+                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+                "Accept-Encoding": "gzip, deflate, br, zstd",
+                "Accept-Language": "en-US,en;q=0.9",
+                "Sec-Ch-Ua": uad,
+                "Sec-Ch-Ua-Mobile": "?0",
+                "Sec-Ch-Ua-Platform": "Windows",
+                "Sec-Fetch-Dest": "document",
+                "Sec-Fetch-Mode": "navigate",
+                "Sec-Fetch-Site": "none",
+                "Sec-Fetch-User": "?1",
+                "Upgrade-Insecure-Requests": "1",
+                "User-Agent": `Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${cmv}.0.0.0 Safari/537.36`
+            };
+            if (cmv >= 124)
+                ch["Priority"] = "u=0, i";
+            return ch;
+        }]
+]);
+class HttpResolverContext {
+    _cmv;
+    _l;
+    _als = new async_hooks_1.AsyncLocalStorage();
+    _mode;
+    _ciphers;
+    _proxyConfig;
+    _chHeaders;
+    _cookies;
+    constructor(_cmv, op, _l = console) {
+        this._cmv = _cmv;
+        this._l = _l;
+        this._mode = op?.mode ?? u_array_1.UArray.randomPick([exports.s_clientMode.chrome, exports.s_clientMode.firefox]);
+        this._ciphers = this.createCiphers(this._mode);
+        this._proxyConfig = op?.proxy;
+        this._chHeaders = s_mode2headers.get(this._mode)(this._cmv);
+        this._l.log(`Context launched with ${Object.keys(exports.s_clientMode).find((_, i) => i === this._mode.id)}:${this._cmv}.`);
+    }
+    /**
+     * request GET to the url.
+     * @param url target url.
+     * @param op.headers http headers.
+     * @param op.ignoreQuery if true, query part in the `url` is ignored.
+     * @returns string encoded by utf-8 as response payload.
+     */
+    async get(url, op) {
+        const u = new url_1.URL(url);
+        const proxyAgent = this._proxyConfig && await this.createProxyAgent(u);
+        const rc = { redirectCount: op?.outerRedirectCount ?? 0, proxyAgent };
+        Object.assign(rc, op);
+        return await this._als.run(rc, this.getIn, u).finally(() => proxyAgent?.destroy());
+    }
+    /**
+     * request POST to the url.
+     * @param url target url.
+     * @param payload request payload. if this is an object, it is treated as json.
+     * @param op.headers http headers.
+     * @param op.ignoreQuery if true, query part in the `url` is ignored.
+     * @returns string encoded by utf-8 as response payload.
+     */
+    async post(url, payload, op) {
+        const u = new url_1.URL(url);
+        const proxyAgent = this._proxyConfig && await this.createProxyAgent(u);
+        const rc = { redirectCount: 0, proxyAgent };
+        Object.assign(rc, op);
+        return await this._als.run(rc, this.postIn, u, payload).finally(() => proxyAgent?.destroy());
+    }
+    createProxyAgent(u) {
+        const conf = this._proxyConfig;
+        return new Promise((resolve, reject) => {
+            const headers = {};
+            if (conf.auth)
+                headers['Proxy-Authorization'] = `Basic ${Buffer.from(conf.auth.name + ':' + conf.auth.pass).toString('base64')}`;
+            const req = (0, http_1.request)({
+                host: conf.server,
+                port: conf.port,
+                method: http_method_1.HttpMethod.Connect,
+                path: `${u.hostname}:443`,
+                headers
+            }).on('connect', (res, socket) => {
+                if (res.statusCode === 200)
+                    resolve(new https_1.Agent({ socket, keepAlive: true }));
+                else
+                    reject(new xjs_err_1.XjsErr(s_errCode, "Could not connect to proxy."));
+            });
+            req.on('error', reject);
+            req.on('timeout', () => {
+                req.destroy();
+                reject(new xjs_err_1.XjsErr(s_errCode, "The http request timeout, maybe server did not respond."));
+            });
+            req.end();
+        });
+    }
+    getIn = async (u) => {
+        const params = {};
+        const rc = this._als.getStore();
+        params.method = http_method_1.HttpMethod.Get;
+        params.headers = rc.headers ?? {};
+        return await this.reqHttps(u, params);
+    };
+    postIn = async (u, payload) => {
+        const params = {};
+        const rc = this._als.getStore();
+        params.method = http_method_1.HttpMethod.Post;
+        params.headers = rc.headers ?? {};
+        let p = payload;
+        if (u_type_1.UType.isObject(payload)) {
+            p = JSON.stringify(payload);
+            params.headers["Content-Length"] = p.length;
+            params.headers["Content-Type"] = "application/json";
+        }
+        return await this.reqHttps(u, params, p);
+    };
+    reqHttps(u, params, payload) {
+        const rc = this._als.getStore();
+        params.timeout = s_timeout;
+        params.protocol = u.protocol;
+        params.host = u.host;
+        params.path = (rc.ignoreQuery || !u.search) ? u.pathname : `${u.pathname}${u.search}`;
+        params.agent = rc.proxyAgent;
+        if (this._mode.id > 0) {
+            params.ciphers = this._ciphers;
+            params.headers = params.headers ? Object.assign(params.headers, this._chHeaders) : this._chHeaders;
+        }
+        if (this._cookies)
+            this.setCookies(params.headers);
+        return new Promise((resolve, reject) => {
+            const req = (0, https_1.request)(params, (res) => {
+                if (res.headers["set-cookie"])
+                    this.storeCookies(res.headers["set-cookie"]);
+                const sc = u_http_1.UHttp.statusCategoryOf(res.statusCode);
+                if (sc === 3) {
+                    this.handleRedirect(res, params.host).then(resolve).catch(reject).finally(() => res.destroy());
+                    return;
+                }
+                const bfs = [];
+                const contentEncofing = res.headers["content-encoding"]?.toLocaleLowerCase();
+                res.on('data', chunk => bfs.push(chunk));
+                res.on('end', () => {
+                    try {
+                        let retBuf = Buffer.concat(bfs);
+                        if (contentEncofing == "gzip")
+                            retBuf = zlib.gunzipSync(retBuf);
+                        else if (contentEncofing == "br")
+                            retBuf = zlib.brotliDecompressSync(retBuf);
+                        const data = retBuf.toString("utf8");
+                        if (sc !== 2) {
+                            if (data.trim())
+                                this.warn(data);
+                            reject(new xjs_err_1.XjsErr(s_errCode, `Https received a error status ${res.statusCode}`));
+                        }
+                        else
+                            resolve(data);
+                    }
+                    catch (e) {
+                        reject(e);
+                    }
+                });
+            });
+            req.on('error', reject);
+            req.on('timeout', () => {
+                req.destroy();
+                reject(new xjs_err_1.XjsErr(s_errCode, "The http request timeout, maybe server did not respond."));
+            });
+            if (payload)
+                req.write(payload);
+            req.end();
+        });
+    }
+    async handleRedirect(res, host) {
+        const rc = this._als.getStore();
+        if (!res.headers.location)
+            throw new xjs_err_1.XjsErr(s_errCode, "Received http redirection, but no location header found.");
+        if (rc.redirectCount++ > s_redirectLimit)
+            throw new xjs_err_1.XjsErr(s_errCode, "Count of http redirection exceeds limit.");
+        this.log(`Redirect to ${res.headers.location}. (count is ${rc.redirectCount})`);
+        const dest = res.headers.location.startsWith("http") ? res.headers.location : `https://${host}${res.headers.location}`;
+        if (rc.outerRedirectCount)
+            throw new xjs_err_1.XjsErr(-1, dest);
+        const u = new url_1.URL(dest);
+        // consider for proxy which implements reverse proxy.
+        if (rc.proxyAgent) {
+            rc.proxyAgent?.destroy();
+            rc.proxyAgent = await this.createProxyAgent(u);
+        }
+        return await this.getIn(u);
+    }
+    createCiphers(mode) {
+        const defaultCiphers = tls.DEFAULT_CIPHERS.split(':');
+        return [
+            defaultCiphers[mode.cipherOrder[0]],
+            defaultCiphers[mode.cipherOrder[1]],
+            defaultCiphers[mode.cipherOrder[2]],
+            ...u_array_1.UArray.shuffle(defaultCiphers.slice(3))
+        ].join(':');
+    }
+    setCookies(headers) {
+        const exp = this._cookies["expires"];
+        if (exp && new Date(exp).getTime() <= Date.now()) {
+            this._cookies = null;
+            this.log("Cookies was cleared due to an expiraion.");
+        }
+        else
+            headers.cookie = Object.keys(this._cookies)
+                .filter(ckk => !["expires", "max-age"].includes(ckk))
+                .map(ckk => `${ckk}=${this._cookies[ckk]};`).join(" ");
+    }
+    storeCookies(cookies) {
+        this._cookies ??= {};
+        cookies.filter(c => c).flatMap(c => c.split(";"))
+            .map(c => {
+            const idx = c.indexOf("=");
+            return idx !== -1 && [c.substring(0, idx).toLowerCase().trim(), c.substring(idx + 1)];
+        })
+            .filter(cp => cp && cp[0] && !["secure", "path", "domain", "samesite"].includes(cp[0]))
+            .forEach(cp => this._cookies[cp[0]] = cp[1]);
+        this.log("Store cookies from set-cookie headers.");
+        this.log(JSON.stringify(this._cookies));
+    }
+    log(msg) {
+        this._l.log(`[http-resolver] ${msg}`);
+    }
+    warn(msg) {
+        this._l.warn(`[http-resolver] ${msg}`);
+    }
+}
+exports.HttpResolverContext = HttpResolverContext;

package/dist/prcs/http/http-resolver.d.ts

@@ -0,0 +1,38 @@
+import { HttpResolverContext } from "./http-resolver-context";
+import { ClientOption, IHttpClient, RequestOption } from "./i-http-client";
+export interface ClientMode {
+    id: number;
+    cipherOrder: number[];
+}
+export interface ProxyConfig {
+    server: string;
+    port: number;
+    auth?: {
+        name: string;
+        pass: string;
+    };
+}
+export declare class HttpResolver implements IHttpClient {
+    private _baseCmv;
+    private _l;
+    /**
+     * @param _baseCmv chrome major version refered when construct a user agent, and the version will be randomized between `n` to `n-4`.
+     * @param _l custom logger. default is `console`.
+     */
+    constructor(_baseCmv: number, _l?: {
+        log: (msg: any) => void;
+        warn: (msg: any) => void;
+    });
+    /**
+     * create a http client as new context that keeps some states. (browser type, cookies, ciphers order, etc...)
+     * @param op.mode {@link s_clientMode} that is imitated. default is random between chrome or firefox.
+     * @param op.proxy proxy configuration.
+     * @returns a http client as new context.
+     */
+    newContext(op?: ClientOption): HttpResolverContext;
+    get(url: string, op?: RequestOption & ClientOption & {
+        redirectAsNewRequest?: boolean;
+    }): Promise<any>;
+    post(url: string, payload: any, op?: RequestOption & ClientOption): Promise<any>;
+    private fixCmv;
+}

package/dist/prcs/http/http-resolver.js

@@ -0,0 +1,52 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HttpResolver = void 0;
+const xjs_err_1 = require("../../obj/xjs-err");
+const http_resolver_context_1 = require("./http-resolver-context");
+const s_cmvRange = 5;
+class HttpResolver {
+    _baseCmv;
+    _l;
+    /**
+     * @param _baseCmv chrome major version refered when construct a user agent, and the version will be randomized between `n` to `n-4`.
+     * @param _l custom logger. default is `console`.
+     */
+    constructor(_baseCmv, _l = console) {
+        this._baseCmv = _baseCmv;
+        this._l = _l;
+    }
+    /**
+     * create a http client as new context that keeps some states. (browser type, cookies, ciphers order, etc...)
+     * @param op.mode {@link s_clientMode} that is imitated. default is random between chrome or firefox.
+     * @param op.proxy proxy configuration.
+     * @returns a http client as new context.
+     */
+    newContext(op) {
+        return new http_resolver_context_1.HttpResolverContext(this.fixCmv(), op, this._l);
+    }
+    async get(url, op) {
+        let redirectCount = op?.redirectAsNewRequest && -1;
+        const bindOp = () => {
+            const option = Object.assign({}, op);
+            if (redirectCount)
+                Object.assign(option, { outerRedirectCount: ++redirectCount });
+            return option;
+        };
+        try {
+            return await this.newContext(op).get(url, bindOp());
+        }
+        catch (e) {
+            if (!(e instanceof xjs_err_1.XjsErr) || e.code !== -1)
+                throw e;
+            else
+                return await this.newContext(op).get(e.message, bindOp());
+        }
+    }
+    async post(url, payload, op) {
+        return await this.newContext(op).post(url, payload, op);
+    }
+    fixCmv() {
+        return this._baseCmv - Math.floor(Math.random() * s_cmvRange);
+    }
+}
+exports.HttpResolver = HttpResolver;

package/dist/prcs/http/i-http-client.d.ts

@@ -0,0 +1,37 @@
+/// <reference types="node" />
+import { OutgoingHttpHeaders } from "http";
+import { ClientMode, ProxyConfig } from "./http-resolver";
+export interface ClientOption {
+    mode?: ClientMode;
+    proxy?: ProxyConfig;
+}
+export interface RequestOption {
+    ignoreQuery?: boolean;
+    headers?: OutgoingHttpHeaders;
+}
+export interface IHttpClient {
+    /**
+     * request GET to the url with new context.
+     * @param url target url.
+     * @param op.headers http headers.
+     * @param op.mode {@link s_clientMode} that is imitated. default is random between chrome or firefox.
+     * @param op.proxy proxy configuration.
+     * @param op.ignoreQuery if true, query part in the `url` is ignored.
+     * @param op.redirectAsNewRequest handle redirect as new request. this may be efficient when using proxy which is implemented reverse proxy.
+     * @returns string encoded by utf-8 as response payload.
+     */
+    get(url: string, op?: RequestOption & ClientOption & {
+        redirectAsNewRequest?: boolean;
+    }): Promise<any>;
+    /**
+     * request POST to the url with new context.
+     * @param url target url.
+     * @param payload request payload. if this is an object, it is treated as json.
+     * @param op.headers http headers.
+     * @param op.mode {@link s_clientMode} that is imitated. default is random between chrome or firefox.
+     * @param op.proxy proxy configuration.
+     * @param op.ignoreQuery if true, query part in the `url` is ignored.
+     * @returns string encoded by utf-8 as response payload.
+     */
+    post(url: string, payload: any, op?: RequestOption & ClientOption): Promise<any>;
+}

package/dist/prcs/http/i-http-client.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/prcs/http-resolver.d.ts

@@ -0,0 +1,53 @@
+export declare const s_clientMode: Record<string, ClientMode>;
+interface ClientMode {
+    id: number;
+    cipherOrder: number[];
+}
+interface ProxyConfig {
+    server: string;
+    port: number;
+    auth?: {
+        name: string;
+        pass: string;
+    };
+}
+export declare class HttpResolver {
+    private _baseCmv;
+    private _l;
+    private readonly _als;
+    private readonly _mode2headers;
+    /**
+     * @param _baseCmv chrome major version refered when construct a user agent, and the version will be randomized between `n` to `n-4`.
+     * @param _l custom logger. default is `console`.
+     */
+    constructor(_baseCmv: number, _l?: {
+        log: (msg: any) => void;
+        warn: (msg: any) => void;
+    });
+    /**
+     * request to the url with GET.
+     * @param url target url.
+     * @param op.mode {@link s_clientMode} that is imitated. default is random between chrome or firefox.
+     * @param op.proxy proxy configuration.
+     * @param op.ignoreQuery if true, query part in the `url` is ignored.
+     * @param op.redirectAsNewRequest handle redirect as new request. this may be efficient when using proxy which is implemented reverse proxy.
+     * @returns string encoded by utf-8 as response payload.
+     */
+    get(url: string, op?: {
+        mode?: ClientMode;
+        ignoreQuery?: boolean;
+        proxy?: ProxyConfig;
+        redirectAsNewRequest?: boolean;
+    }): Promise<any>;
+    private fixCmv;
+    private createProxyAgent;
+    private getIn;
+    private reqHttps;
+    private handleRedirect;
+    private createCiphers;
+    private setCookies;
+    private storeCookies;
+    private log;
+    private warn;
+}
+export {};