xjs-node 1.0.6 → 2.0.0

package/dist/prcs/http/http-resolver-context.js

@@ -1,326 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.HttpResolverContext = exports.s_clientMode = void 0;
-const tls = __importStar(require("tls"));
-const zlib = __importStar(require("zlib"));
-const fs = __importStar(require("fs"));
-const path = __importStar(require("path"));
-const url_1 = require("url");
-const https_1 = require("https");
-const http_1 = require("http");
-const async_hooks_1 = require("async_hooks");
-const u_file_1 = require("../../func/u-file");
-const u_1 = require("../../func/u");
-const xjs_common_1 = require("xjs-common");
-const stream_1 = require("stream");
-exports.s_clientMode = {
-    nodejs: { id: 0, cipherOrder: null },
-    chrome: { id: 1, cipherOrder: [2, 0, 1] },
-    firefox: { id: 2, cipherOrder: [2, 1, 0] }
-};
-const s_errCode = 1200;
-const s_redirectLimit = 5;
-const s_mode2headers = new Map([
-    [exports.s_clientMode.firefox, (cmv) => ({
-            "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-            "accept-encoding": "gzip, deflate, br",
-            "accept-language": "en-US,en;q=0.5",
-            "sec-fetch-dest": "document",
-            "sec-fetch-mode": "navigate",
-            "sec-fetch-site": "none",
-            "sec-fetch-user": "?1",
-            "upgrade-insecure-requests": "1",
-            "user-agent": `Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:${cmv}.0) Gecko/20100101 Firefox/${cmv}.0`
-        })],
-    [exports.s_clientMode.chrome, (cmv) => {
-            const uad = cmv < 130
-                ? `"Not/A)Brand";v="8", "Chromium";v="${cmv}", "Google Chrome";v="${cmv}"`
-                : `"Chromium";v="${cmv}", "Not:A-Brand";v="24", "Google Chrome";v="${cmv}"`;
-            const ch = {
-                "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
-                "accept-encoding": "gzip, deflate, br, zstd",
-                "accept-language": "en-US,en;q=0.9",
-                "sec-ch-ua": uad,
-                "sec-ch-ua-mobile": "?0",
-                "sec-ch-ua-platform": '"Windows"',
-                "sec-fetch-dest": "document",
-                "sec-fetch-mode": "navigate",
-                "sec-fetch-site": "none",
-                "sec-fetch-user": "?1",
-                "upgrade-insecure-requests": "1",
-                "user-agent": `Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${cmv}.0.0.0 Safari/537.36`
-            };
-            if (cmv >= 124)
-                ch["priority"] = "u=0, i";
-            return ch;
-        }]
-]);
-class HttpResolverContext {
-    cmv;
-    _l;
-    _als = new async_hooks_1.AsyncLocalStorage();
-    _mode;
-    _ciphers;
-    _proxyConfig;
-    _chHeaders;
-    _cookies;
-    get clientMode() { return Object.keys(exports.s_clientMode).find((_, i) => i === this._mode.id); }
-    constructor(cmv, op, _l = console) {
-        this.cmv = cmv;
-        this._l = _l;
-        this._mode = op?.mode ?? xjs_common_1.UArray.randomPick([exports.s_clientMode.chrome, exports.s_clientMode.firefox]);
-        this._proxyConfig = op?.proxy;
-        if (this._mode.id > 0) {
-            this._ciphers = this.createCiphers(this._mode);
-            this._chHeaders = s_mode2headers.get(this._mode)(this.cmv);
-        }
-    }
-    async get(url, op) {
-        const u = new url_1.URL(url);
-        const proxyAgent = this._proxyConfig && await this.createProxyAgent(u);
-        const rc = { redirectCount: op?.outerRedirectCount ?? 0, proxyAgent };
-        Object.assign(rc, op);
-        return await this._als.run(rc, this.getIn, u).finally(() => proxyAgent?.destroy());
-    }
-    async post(url, payload, op) {
-        const u = new url_1.URL(url);
-        const proxyAgent = this._proxyConfig && await this.createProxyAgent(u);
-        const rc = { redirectCount: 0, proxyAgent };
-        Object.assign(rc, op);
-        return await this._als.run(rc, this.postIn, u, payload).finally(() => proxyAgent?.destroy());
-    }
-    createProxyAgent(u) {
-        const conf = this._proxyConfig;
-        return new Promise((resolve, reject) => {
-            const headers = {};
-            if (conf.auth)
-                headers['proxy-authorization'] = `Basic ${Buffer.from(conf.auth.name + ':' + conf.auth.pass).toString('base64')}`;
-            const req = (0, http_1.request)({
-                host: conf.server,
-                port: conf.port,
-                method: xjs_common_1.HttpMethod.Connect,
-                path: `${u.hostname}:443`,
-                headers
-            }).on('connect', (res, socket) => {
-                if (res.statusCode === 200)
-                    resolve(new https_1.Agent({ socket, keepAlive: true }));
-                else
-                    reject(new xjs_common_1.XjsErr(s_errCode, "Could not connect to proxy."));
-            });
-            req.on('error', reject);
-            req.on('timeout', () => {
-                req.destroy();
-                reject(new xjs_common_1.XjsErr(s_errCode, "The http request timeout, maybe server did not respond."));
-            });
-            req.end();
-        });
-    }
-    getIn = async (u) => {
-        const params = {};
-        const rc = this._als.getStore();
-        params.method = xjs_common_1.HttpMethod.Get;
-        params.headers = xjs_common_1.UHttp.normalizeHeaders(rc.headers);
-        return await this.reqHttps(u, params);
-    };
-    postIn = async (u, payload) => {
-        const params = {};
-        const rc = this._als.getStore();
-        params.method = xjs_common_1.HttpMethod.Post;
-        params.headers = xjs_common_1.UHttp.normalizeHeaders(rc.headers);
-        let p = payload;
-        if (p instanceof stream_1.Stream) {
-            params.headers["content-type"] ??= "application/octet-stream";
-        }
-        else if (xjs_common_1.UType.isObject(payload)) {
-            p = JSON.stringify(payload);
-            params.headers["content-length"] = p.length;
-            params.headers["content-type"] = "application/json";
-        }
-        return await this.reqHttps(u, params, p);
-    };
-    reqHttps(u, params, payload) {
-        const rc = this._als.getStore();
-        params.timeout = rc.timeout ?? 0;
-        params.protocol = u.protocol;
-        params.host = u.host;
-        params.path = (rc.ignoreQuery || !u.search) ? u.pathname : `${u.pathname}${u.search}`;
-        params.agent = rc.proxyAgent;
-        if (this._mode.id > 0) {
-            params.ciphers = this._ciphers;
-            params.headers = params.headers ? Object.assign(params.headers, this._chHeaders) : this._chHeaders;
-        }
-        if (this._cookies)
-            this.setCookies(params.headers);
-        return new Promise((resolve, reject) => {
-            const req = (0, https_1.request)(params, (res) => this.processResponse(resolve, reject, rc, params.host, res));
-            req.on('error', reject);
-            req.on('timeout', () => {
-                req.destroy();
-                reject(new xjs_common_1.XjsErr(s_errCode, "The http request timeout, maybe server did not respond."));
-            });
-            if (payload)
-                if (payload instanceof stream_1.Stream)
-                    payload.pipe(req);
-                else
-                    req.write(payload);
-            req.end();
-        });
-    }
-    processResponse(resolve, reject, rc, host, res) {
-        if (res.headers["set-cookie"])
-            this.storeCookies(res.headers["set-cookie"]);
-        const sc = xjs_common_1.UHttp.statusCategoryOf(res.statusCode);
-        if (sc === 3) {
-            this.handleRedirect(res, host).then(resolve).catch(reject).finally(() => res.destroy());
-            return;
-        }
-        if (res.headers["content-disposition"]?.trim().startsWith("attachment")) {
-            try {
-                const dest = this.resolveDownloadPath(rc.downloadPath, res.headers["content-disposition"]);
-                const stream = fs.createWriteStream(dest);
-                res.pipe(stream);
-                stream.on("finish", () => stream.close());
-                stream.on("close", () => resolve({ headers: res.headers }));
-                stream.on("error", reject);
-                return;
-            }
-            catch (e) {
-                if (e instanceof xjs_common_1.XjsErr)
-                    reject(e);
-                else {
-                    this.error(e);
-                    reject(new xjs_common_1.XjsErr(s_errCode, "Failed to download a file."));
-                }
-            }
-        }
-        const bfs = [];
-        const contentEncofing = res.headers["content-encoding"]?.toLocaleLowerCase();
-        res.on('data', chunk => bfs.push(chunk));
-        res.on('end', () => {
-            try {
-                let retBuf = Buffer.concat(bfs);
-                if (contentEncofing == "gzip")
-                    retBuf = zlib.gunzipSync(retBuf);
-                else if (contentEncofing == "br")
-                    retBuf = zlib.brotliDecompressSync(retBuf);
-                const data = rc.responseType === "buffer" ? retBuf : retBuf.toString("utf8");
-                if (sc !== 2) {
-                    if (xjs_common_1.UType.isString(data) && data.trim())
-                        this.warn(data);
-                    reject(new xjs_common_1.XjsErr(s_errCode, `Https received a error status ${res.statusCode}`));
-                }
-                else
-                    resolve({ payload: data, headers: res.headers });
-            }
-            catch (e) {
-                reject(e);
-            }
-        });
-    }
-    resolveDownloadPath(opPath, disposition) {
-        const appendFname = (d) => {
-            const fname = disposition.split(";")
-                .find(f => f.trim().startsWith("filename"))
-                ?.replace(/^\s+filename\s+=/, "").trim()
-                ?? u_file_1.UFile.reserveFilePath(d, `xjs-download_${xjs_common_1.UString.simpleTime()}`);
-            return (0, u_1.joinPath)(d, fname);
-        };
-        if (opPath) {
-            const st = u_file_1.UFile.status(opPath);
-            if (!st || st.isFile()) {
-                if (!u_file_1.UFile.exists(path.dirname(opPath)))
-                    throw new xjs_common_1.XjsErr(s_errCode, "Directory of the download file was not found.");
-                return opPath;
-            }
-            if (st.isDirectory()) {
-                if (!u_file_1.UFile.exists(opPath))
-                    throw new xjs_common_1.XjsErr(s_errCode, "Directory of the download path was not found.");
-                return appendFname(opPath);
-            }
-        }
-        return appendFname("./");
-    }
-    async handleRedirect(res, host) {
-        const rc = this._als.getStore();
-        if (!res.headers.location)
-            throw new xjs_common_1.XjsErr(s_errCode, "Received http redirection, but no location header found.");
-        if (rc.redirectCount++ > s_redirectLimit)
-            throw new xjs_common_1.XjsErr(s_errCode, "Count of http redirection exceeds limit.");
-        this.log(`Redirect to ${res.headers.location}. (count is ${rc.redirectCount})`);
-        const dest = res.headers.location.startsWith("http") ? res.headers.location : `https://${host}${res.headers.location}`;
-        if (rc.outerRedirectCount)
-            throw new xjs_common_1.XjsErr(-1, dest);
-        const u = new url_1.URL(dest);
-        // consider for proxy which implements reverse proxy.
-        if (rc.proxyAgent) {
-            rc.proxyAgent?.destroy();
-            rc.proxyAgent = await this.createProxyAgent(u);
-        }
-        return await this.getIn(u);
-    }
-    createCiphers(mode) {
-        const defaultCiphers = tls.DEFAULT_CIPHERS.split(':');
-        return [
-            defaultCiphers[mode.cipherOrder[0]],
-            defaultCiphers[mode.cipherOrder[1]],
-            defaultCiphers[mode.cipherOrder[2]],
-            ...xjs_common_1.UArray.shuffle(defaultCiphers.slice(3))
-        ].join(':');
-    }
-    setCookies(headers) {
-        const exp = this._cookies["expires"];
-        if (exp && new Date(exp).getTime() <= Date.now()) {
-            this._cookies = null;
-            this.log("Cookies was cleared due to an expiraion.");
-        }
-        else
-            headers.cookie = Object.keys(this._cookies)
-                .filter(ckk => !["expires", "max-age"].includes(ckk))
-                .map(ckk => `${ckk}=${this._cookies[ckk]};`).join(" ");
-    }
-    storeCookies(cookies) {
-        this._cookies ??= {};
-        cookies.filter(c => c).flatMap(c => c.split(";"))
-            .map(c => {
-            const idx = c.indexOf("=");
-            return idx !== -1 && [c.substring(0, idx).toLowerCase().trim(), c.substring(idx + 1)];
-        })
-            .filter(cp => cp && cp[0] && !["secure", "path", "domain", "samesite"].includes(cp[0]))
-            .forEach(cp => this._cookies[cp[0]] = cp[1]);
-        this.log("Store cookies from set-cookie headers.");
-        this.log(JSON.stringify(this._cookies));
-    }
-    log(msg) {
-        this._l.log(`[http-resolver] ${msg}`);
-    }
-    warn(msg) {
-        this._l.warn(`[http-resolver] ${msg}`);
-    }
-    error(msg) {
-        this._l.error(`[http-resolver] ${msg}`);
-    }
-}
-exports.HttpResolverContext = HttpResolverContext;

package/dist/prcs/http/http-resolver.d.ts

@@ -1,51 +0,0 @@
-/// <reference types="node" />
-import { Loggable } from "xjs-common";
-import { HttpResolverContext } from "./http-resolver-context";
-import { ClientOption, HttpResponse, IHttpClient, RequestOption } from "./i-http-client";
-export interface ClientMode {
-    id: number;
-    cipherOrder: number[];
-}
-export interface ProxyConfig {
-    server: string;
-    port: number;
-    auth?: {
-        name: string;
-        pass: string;
-    };
-}
-export declare class HttpResolver implements IHttpClient {
-    private _baseCmv;
-    private _l;
-    /**
-     * @param _baseCmv chrome major version refered when construct a user agent, and the version will be randomized between `n` to `n-4`.
-     * @param _l custom logger. default is `console`.
-     */
-    constructor(_baseCmv?: number, _l?: Loggable);
-    /**
-     * create a http client as new context that keeps some states. (browser type, cookies, ciphers order, etc...)
-     * @param op.mode {@link s_clientMode} that is imitated. default is random between chrome or firefox.
-     * @param op.proxy proxy configuration.
-     * @returns a http client as new context.
-     */
-    newContext(op?: ClientOption): HttpResolverContext;
-    get(url: string, op?: RequestOption & ClientOption & {
-        redirectAsNewRequest?: boolean;
-        responseType: "string";
-    }): Promise<HttpResponse<string>>;
-    get(url: string, op?: RequestOption & ClientOption & {
-        redirectAsNewRequest?: boolean;
-        responseType: "buffer";
-    }): Promise<HttpResponse<Buffer>>;
-    get(url: string, op?: RequestOption & ClientOption & {
-        redirectAsNewRequest?: boolean;
-    }): Promise<HttpResponse<string>>;
-    post(url: string, payload: any, op?: RequestOption & ClientOption & {
-        responseType: "string";
-    }): Promise<HttpResponse<string>>;
-    post(url: string, payload: any, op?: RequestOption & ClientOption & {
-        responseType: "buffer";
-    }): Promise<HttpResponse<Buffer>>;
-    post(url: string, payload: any, op?: RequestOption & ClientOption): Promise<HttpResponse<string>>;
-    private fixCmv;
-}

package/dist/prcs/http/http-resolver.js

@@ -1,53 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.HttpResolver = void 0;
-const xjs_common_1 = require("xjs-common");
-const http_resolver_context_1 = require("./http-resolver-context");
-const s_cmvRange = 5;
-const s_defaultCmv = 138;
-class HttpResolver {
-    _baseCmv;
-    _l;
-    /**
-     * @param _baseCmv chrome major version refered when construct a user agent, and the version will be randomized between `n` to `n-4`.
-     * @param _l custom logger. default is `console`.
-     */
-    constructor(_baseCmv = s_defaultCmv, _l = console) {
-        this._baseCmv = _baseCmv;
-        this._l = _l;
-    }
-    /**
-     * create a http client as new context that keeps some states. (browser type, cookies, ciphers order, etc...)
-     * @param op.mode {@link s_clientMode} that is imitated. default is random between chrome or firefox.
-     * @param op.proxy proxy configuration.
-     * @returns a http client as new context.
-     */
-    newContext(op) {
-        return new http_resolver_context_1.HttpResolverContext(this.fixCmv(), op, this._l);
-    }
-    async get(url, op) {
-        let redirectCount = op?.redirectAsNewRequest && -1;
-        const bindOp = () => {
-            const option = Object.assign({}, op);
-            if (redirectCount)
-                Object.assign(option, { outerRedirectCount: ++redirectCount });
-            return option;
-        };
-        try {
-            return await this.newContext(op).get(url, bindOp());
-        }
-        catch (e) {
-            if (!(e instanceof xjs_common_1.XjsErr) || e.code !== -1)
-                throw e;
-            else
-                return await this.newContext(op).get(e.message, bindOp());
-        }
-    }
-    async post(url, payload, op) {
-        return await this.newContext(op).post(url, payload, op);
-    }
-    fixCmv() {
-        return this._baseCmv - Math.floor(Math.random() * s_cmvRange);
-    }
-}
-exports.HttpResolver = HttpResolver;

package/dist/prcs/http/i-http-client.d.ts

@@ -1,90 +0,0 @@
-/// <reference types="node" />
-/// <reference types="node" />
-import { IncomingHttpHeaders, OutgoingHttpHeaders } from "http";
-import { ClientMode, ProxyConfig } from "./http-resolver";
-export interface ClientOption {
-    /**
-     * {@link s_clientMode} that is imitated. default is random between chrome or firefox.
-     */
-    mode?: ClientMode;
-    /**
-     * proxy configuration.
-     */
-    proxy?: ProxyConfig;
-}
-export interface RequestOption {
-    headers?: OutgoingHttpHeaders;
-    /**
-     * if true, query part in the `url` is ignored.
-     */
-    ignoreQuery?: boolean;
-    /**
-     * destination directory or file path for download. this is only used when `Content-Disposition` header exists.
-     * default is current directory of the process with `filename` of the disposition.
-     */
-    downloadPath?: string;
-    /**
-     * timeout milliseconds to wait for socket inactivity. default is infinity.
-     */
-    timeout?: number;
-    /**
-     * type of response payload. default is string (utf-8).
-     */
-    responseType?: "string" | "buffer";
-}
-export interface HttpResponse<T = string | Buffer> {
-    /**
-     * http headers in the response.
-     */
-    headers?: IncomingHttpHeaders;
-    /**
-     * response payload which has a type depends on {@link RequestOption.responseType}.
-     */
-    payload?: T;
-}
-export interface IHttpClient {
-    /**
-     * request GET to the url with new context.
-     * @param url target url. (currently https only)
-     * @param op.headers http headers.
-     * @param op.mode {@link s_clientMode} that is imitated. default is random between chrome or firefox.
-     * @param op.proxy proxy configuration.
-     * @param op.ignoreQuery {@link RequestOption.ignoreQuery}
-     * @param op.downloadPath {@link RequestOption.downloadPath}
-     * @param op.timeout {@link RequestOption.timeout}
-     * @param op.responseType {@link RequestOption.responseType}
-     * @param op.redirectAsNewRequest handle redirect as new request. this may be efficient when using proxy which is implemented reverse proxy.
-     * @returns http response. {@link HttpResponse}
-    */
-    get(url: string, op?: RequestOption & ClientOption & {
-        redirectAsNewRequest?: boolean;
-        responseType: "string";
-    }): Promise<HttpResponse<string>>;
-    get(url: string, op?: RequestOption & ClientOption & {
-        redirectAsNewRequest?: boolean;
-        responseType: "buffer";
-    }): Promise<HttpResponse<Buffer>>;
-    get(url: string, op?: RequestOption & ClientOption & {
-        redirectAsNewRequest?: boolean;
-    }): Promise<HttpResponse<string>>;
-    /**
-     * request POST to the url with new context.
-     * @param url target url. (currently https only)
-     * @param payload request payload. if this is a Stream, pipe will be used, otherwise if an object, this is treated as json.
-     * @param op.headers http headers.
-     * @param op.mode {@link s_clientMode} that is imitated. default is random between chrome or firefox.
-     * @param op.proxy proxy configuration.
-     * @param op.ignoreQuery {@link RequestOption.ignoreQuery}
-     * @param op.downloadPath {@link RequestOption.downloadPath}
-     * @param op.timeout {@link RequestOption.timeout}
-     * @param op.responseType {@link RequestOption.responseType}
-     * @returns http response. {@link HttpResponse}
-     */
-    post(url: string, payload: any, op?: RequestOption & ClientOption & {
-        responseType: "string";
-    }): Promise<HttpResponse<string>>;
-    post(url: string, payload: any, op?: RequestOption & ClientOption & {
-        responseType: "buffer";
-    }): Promise<HttpResponse<Buffer>>;
-    post(url: string, payload: any, op?: RequestOption & ClientOption): Promise<HttpResponse<string>>;
-}

package/dist/prcs/http/i-http-client.js

@@ -1,2 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });