xjs-common 7.0.0

package/dist/prcs/http-resolver.js

@@ -0,0 +1,255 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HttpResolver = exports.s_clientMode = void 0;
+const tls = __importStar(require("tls"));
+const zlib = __importStar(require("zlib"));
+const url_1 = require("url");
+const https_1 = require("https");
+const http_1 = require("http");
+const async_hooks_1 = require("async_hooks");
+const xjs_err_1 = require("../obj/xjs-err");
+const u_http_1 = require("../func/u-http");
+const u_array_1 = require("../func/u-array");
+exports.s_clientMode = {
+    nodejs: { id: 0, cipherOrder: null },
+    chrome: { id: 1, cipherOrder: [2, 0, 1] },
+    firefox: { id: 2, cipherOrder: [2, 1, 0] }
+};
+const s_cmvRange = 5;
+const s_timeout = 1000 * 20;
+const s_errCode = 200;
+class HttpResolver {
+    _baseCmv;
+    _l;
+    _als = new async_hooks_1.AsyncLocalStorage();
+    _mode2headers = new Map([
+        [exports.s_clientMode.firefox, (cmv) => ({
+                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+                "Accept-Encoding": "gzip, deflate, br",
+                "Accept-Language": "en-US,en;q=0.5",
+                "Sec-Fetch-Dest": "document",
+                "Sec-Fetch-Mode": "navigate",
+                "Sec-Fetch-Site": "none",
+                "Sec-Fetch-User": "?1",
+                "Upgrade-Insecure-Requests": "1",
+                "User-Agent": `Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:${cmv}.0) Gecko/20100101 Firefox/${cmv}.0`
+            })],
+        [exports.s_clientMode.chrome, (cmv) => {
+                const uad = cmv < 130
+                    ? `"Not/A)Brand";v="8", "Chromium";v="${cmv}", "Google Chrome";v="${cmv}"`
+                    : `"Chromium";v="${cmv}", "Not:A-Brand";v="24", "Google Chrome";v="${cmv}"`;
+                const ch = {
+                    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+                    "Accept-Encoding": "gzip, deflate, br, zstd",
+                    "Accept-Language": "en-US,en;q=0.9",
+                    "Sec-Ch-Ua": uad,
+                    "Sec-Ch-Ua-Mobile": "?0",
+                    "Sec-Ch-Ua-Platform": "Windows",
+                    "Sec-Fetch-Dest": "document",
+                    "Sec-Fetch-Mode": "navigate",
+                    "Sec-Fetch-Site": "none",
+                    "Sec-Fetch-User": "?1",
+                    "Upgrade-Insecure-Requests": "1",
+                    "User-Agent": `Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${cmv}.0.0.0 Safari/537.36`
+                };
+                if (cmv >= 124)
+                    ch["Priority"] = "u=0, i";
+                return ch;
+            }]
+    ]);
+    /**
+     * @param _baseCmv chrome major version refered when construct a user agent, and the version will be randomized between `n` to `n-4`.
+     * @param _l custom logger. default is `console`.
+     */
+    constructor(_baseCmv, _l = console) {
+        this._baseCmv = _baseCmv;
+        this._l = _l;
+    }
+    /**
+     * request to the url with GET.
+     * @param url target url.
+     * @param op.mode {@link s_clientMode} that is imitated. default is random between chrome or firefox.
+     * @param op.proxy proxy configuration.
+     * @param op.ignoreQuery if true, query part in the `url` is ignored.
+     * @param op.redirectAsNewRequest handle redirect as new request. this may be efficient when using proxy which is implemented reverse proxy.
+     * @returns string encoded by utf-8 as response payload.
+     */
+    async get(url, op) {
+        const cmv = this.fixCmv();
+        const m = op?.mode ?? u_array_1.UArray.randomPick([exports.s_clientMode.chrome, exports.s_clientMode.firefox]);
+        const ciphers = this.createCiphers(m);
+        const u = new url_1.URL(url);
+        const proxyAgent = op?.proxy && await this.createProxyAgent(u, op.proxy);
+        this._l.log(`Starts to request with ${Object.keys(exports.s_clientMode).find((_, i) => i === m.id)}:${cmv}.`);
+        const rc = {
+            mode: m, cmv, ciphers,
+            ignoreQuery: !!op?.ignoreQuery,
+            redirectCount: 0, redirectAsNewRequest: !!op?.redirectAsNewRequest,
+            proxyAgent, proxyConfig: op?.proxy
+        };
+        return await this._als.run(rc, this.getIn, u).finally(() => proxyAgent?.destroy());
+    }
+    fixCmv() {
+        return this._baseCmv - Math.floor(Math.random() * s_cmvRange);
+    }
+    createProxyAgent(u, conf) {
+        return new Promise((resolve, reject) => {
+            const headers = {};
+            if (conf.auth)
+                headers['Proxy-Authorization'] = `Basic ${Buffer.from(conf.auth.name + ':' + conf.auth.pass).toString('base64')}`;
+            const req = (0, http_1.request)({
+                host: conf.server,
+                port: conf.port,
+                method: 'CONNECT',
+                path: `${u.hostname}:443`,
+                headers
+            }).on('connect', (res, socket) => {
+                if (res.statusCode === 200)
+                    resolve(new https_1.Agent({ socket, keepAlive: true }));
+                else
+                    reject(new xjs_err_1.XjsErr(s_errCode, "Could not connect to proxy."));
+            });
+            req.on('error', reject);
+            req.on('timeout', () => {
+                req.destroy();
+                reject(new xjs_err_1.XjsErr(s_errCode, "The http request timeout, maybe server did not respond."));
+            });
+            req.end();
+        });
+    }
+    getIn = async (u) => {
+        const params = {};
+        const rc = this._als.getStore();
+        const toRefresh = rc.redirectCount > 0 && rc.redirectAsNewRequest;
+        params.method = "GET";
+        params.protocol = u.protocol;
+        params.host = u.host;
+        params.path = (rc.ignoreQuery || !u.search) ? u.pathname : `${u.pathname}${u.search}`;
+        params.agent = toRefresh ? await this.createProxyAgent(u, rc.proxyConfig) : rc.proxyAgent;
+        if (toRefresh) {
+            rc.cmv = this.fixCmv();
+            rc.ciphers = this.createCiphers(rc.mode);
+        }
+        return await this.reqHttps(params, null);
+    };
+    reqHttps(params, postData) {
+        params.timeout = s_timeout;
+        const rc = this._als.getStore();
+        if (rc.mode.id > 0) {
+            params.ciphers = rc.ciphers;
+            const chHeaders = this._mode2headers.get(rc.mode)(rc.cmv);
+            params.headers = params.headers ? Object.assign(params.headers, chHeaders) : chHeaders;
+        }
+        if (rc.cookies && !rc.redirectAsNewRequest)
+            this.setCookies(params.headers);
+        return new Promise((resolve, reject) => {
+            const req = (0, https_1.request)(params, (res) => {
+                const sc = u_http_1.UHttp.statusCategoryOf(res.statusCode);
+                if (sc === 3) {
+                    this.handleRedirect(res, params.host).then(resolve).catch(reject).finally(() => res.destroy());
+                    return;
+                }
+                const bfs = [];
+                const contentEncofing = res.headers["content-encoding"]?.toLocaleLowerCase();
+                res.on('data', chunk => bfs.push(chunk));
+                res.on('end', () => {
+                    try {
+                        let retBuf = Buffer.concat(bfs);
+                        if (contentEncofing == "gzip")
+                            retBuf = zlib.gunzipSync(retBuf);
+                        else if (contentEncofing == "br")
+                            retBuf = zlib.brotliDecompressSync(retBuf);
+                        const data = retBuf.toString("utf8");
+                        if (sc !== 2) {
+                            if (data.trim())
+                                this.warn(data);
+                            reject(new xjs_err_1.XjsErr(s_errCode, `Https received a error status ${res.statusCode}`));
+                        }
+                        else
+                            resolve(data);
+                    }
+                    catch (e) {
+                        reject(e);
+                    }
+                });
+            });
+            req.on('error', reject);
+            req.on('timeout', () => {
+                req.destroy();
+                reject(new xjs_err_1.XjsErr(s_errCode, "The http request timeout, maybe server did not respond."));
+            });
+            if (postData)
+                req.write(postData);
+            req.end();
+        });
+    }
+    handleRedirect(res, host) {
+        const rc = this._als.getStore();
+        if (!res.headers.location)
+            throw new xjs_err_1.XjsErr(s_errCode, "Received http redirection, but no location header found.");
+        if (rc.redirectCount++ > 2)
+            throw new xjs_err_1.XjsErr(s_errCode, "Count of http redirection exceeds limit.");
+        if (res.headers["set-cookie"] && !rc.redirectAsNewRequest)
+            this.storeCookies(res.headers["set-cookie"]);
+        this.log(`Redirect to ${res.headers.location}. (count is ${rc.redirectCount})`);
+        const dest = res.headers.location.startsWith("http") ? res.headers.location : `https://${host}${res.headers.location}`;
+        if (rc.redirectAsNewRequest)
+            rc.proxyAgent?.destroy();
+        return this.getIn(new url_1.URL(dest));
+    }
+    createCiphers(mode) {
+        const defaultCiphers = tls.DEFAULT_CIPHERS.split(':');
+        return [
+            defaultCiphers[mode.cipherOrder[0]],
+            defaultCiphers[mode.cipherOrder[1]],
+            defaultCiphers[mode.cipherOrder[2]],
+            ...u_array_1.UArray.shuffle(defaultCiphers.slice(3))
+        ].join(':');
+    }
+    setCookies(headers) {
+        const cks = this._als.getStore().cookies;
+        headers.cookie = Object.keys(cks).map(ckk => `${ckk}=${cks[ckk]};`).join(" ");
+    }
+    storeCookies(cookies) {
+        this._als.getStore().cookies ??= {};
+        cookies.filter(c => c).flatMap(c => c.split(";"))
+            .map(c => {
+            const idx = c.indexOf("=");
+            return idx !== -1 && [c.substring(0, idx).toLowerCase().trim(), c.substring(idx + 1)];
+        })
+            .filter(cp => cp && cp[0] && !["secure", "path", "domain", "samesite"].includes(cp[0]))
+            .forEach(cp => this._als.getStore().cookies[cp[0]] = cp[1]);
+        this.log("Store cookies from set-cookie headers.");
+        this.log(JSON.stringify(this._als.getStore().cookies));
+    }
+    log(msg) {
+        this._l.log(`[http-resolver] ${msg}`);
+    }
+    warn(msg) {
+        this._l.warn(`[http-resolver] ${msg}`);
+    }
+}
+exports.HttpResolver = HttpResolver;

package/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "xjs-common",
+  "version": "7.0.0",
+  "description": "library modules for nodejs that bundled general-purpose implementations.",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/begyyal/xjs_commons"
+  },
+  "keywords": [
+    "node",
+    "utility"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "test": "node ./dist/test/test.js"
+  },
+  "author": "begyyal",
+  "license": "Apache-2.0",
+  "files": [
+    "dist",
+    "!/dist/test*"
+  ],
+  "main": "./dist/index",
+  "types": "./dist/index.d.ts",
+  "devDependencies": {
+    "@types/node": "^20.3.1",
+    "ts-node": "^10.9.1",
+    "typescript": "^4.9.5"
+  }
+}