xjs-common 8.4.1 → 9.0.0

package/README.md

@@ -11,7 +11,7 @@ npm i xjs-common
 **NOTE**: The versions <= `v6.2.0` was unpublished. If you has been used these versions, please update to the version >= `v7.0.0`.
 
 # Code example (only part)
- - Miscellaneous utilities.
+### Miscellaneous utilities.
 ```ts
 import { checkPortAvailability, delay, int2array, UFile, UHttp, retry } from "xjs-common";
 
@@ -41,7 +41,7 @@ import { checkPortAvailability, delay, int2array, UFile, UHttp, retry } from "xj
     UFile.mkdir(["path", "to", "dir"]);
 })();
 ```
- - Array utilities.
+### Array utilities.
 ```ts
 import { UArray } from "xjs-common";
 
@@ -70,7 +70,7 @@ import { UArray } from "xjs-common";
     console.log(UArray.randomPick(ary3));
 })();
 ```
- - String utilities.
+### String utilities.
 ```ts
 import { UString } from "xjs-common";
 
@@ -94,35 +94,38 @@ import { UString } from "xjs-common";
     console.log(UString.simpleTime({ date: getJSTDate(), unit: TimeUnit.Day }));
 })();
 ```
- - Http client enhanced for web scraping.
+### Enhanced http client.
 ```ts
 import { HttpResolver, s_clientMode } from "xjs-common";
 
 (async () => {
-    const chromeMajorVersion = 134;
     // can customize logging. (default is console.)
     // const http = new HttpResolver(chromeMajorVersion, logger);
-    const http = new HttpResolver(chromeMajorVersion);
+    const http = new HttpResolver();
 
-    // switch tls ciphers order pattern by passing clientMode. (default is random between chrome or firefox.)
-    let body = await http.get("https://begyyal.net", { mode: s_clientMode.chrome });
+    // implicitly corresponds to cookies and redirect, and do randomization.
+    let res = await http.get("https://begyyal.net");
+    const { payload, headers } = res;
 
     // use proxy by passing the configuration.
     const proxy = { server: "proxy.sample.com", port: 8080, auth: { name: "prx", pass: "****" } }
-    body = await http.post("https://begyyal.net", { proxy });
+    res = await http.post("https://begyyal.net", { proxy });
 
-    // implicitly corresponds to cookies and redirect, and do randomization.
-    body = await http.get("https://begyyal.net");
+    // switch tls ciphers order pattern by passing clientMode. (default is random between chrome or firefox.)
+    res = await http.get("https://begyyal.net", { mode: s_clientMode.chrome });
+
+    // download a file when [Content-Disposition: attachment] exists in the response.
+    await http.get("https://begyyal.net/a.txt", { downloadPath: "/path/to/store" });
 
     // if you want to keep some states of requests (and suppress to randomize), it can create new context to do.
     const context = http.newContext();
-    body = await context.get("https://begyyal.net/1");
+    res = await context.get("https://begyyal.net/1");
     // this request sends with cookies that is set by precedent requests. 
     // in POST, payload is treated as json if it is an object.
-    body = await context.post("https://begyyal.net/2", { a: "b" });
+    res = await context.post("https://begyyal.net/2", { a: "b" });
 })();
 ```
- - Mark method as transaction.  
+### Mark method as transaction.  
 **NOTE**: this feature uses decorator, so requires `"experimentalDecorators": true` in tsconfig.
 ```ts
 import { transaction, delay } from "xjs-common";
@@ -147,7 +150,7 @@ class Cls {
     console.log(e);
 });
 ```
- - Validate and crop class fields.  
+### Validate and crop class fields.  
 **NOTE**: this feature uses decorator, so requires `"experimentalDecorators": true` in tsconfig.  
 **NOTE**: some functionalities  in this feature are based on `"useDefineForClassFields": true` in tsconfig.  
 this flag is true by default at the target higher than `ES2022`, [here is for more](https://www.typescriptlang.org/tsconfig/#useDefineForClassFields).

package/dist/func/u-file.d.ts

@@ -1,14 +1,39 @@
 /// <reference types="node" />
 import { MaybeArray } from "../const/types";
+interface FileStatus {
+    isFile(): boolean;
+    isDirectory(): boolean;
+    isBlockDevice(): boolean;
+    isCharacterDevice(): boolean;
+    isSymbolicLink(): boolean;
+    isFIFO(): boolean;
+    isSocket(): boolean;
+}
 export declare namespace UFile {
     function mkdir(p: MaybeArray<string>): boolean;
     function write(p: MaybeArray<string>, c: string): void;
+    /**
+     * remove a file. no error if the file to be removed doesn't exist.
+     */
     function rm(p: MaybeArray<string>): void;
     function exists(p: MaybeArray<string>): boolean;
+    /**
+     * return a file status. if the file of the status doesn't exist, this returns `null`.
+     */
+    function status(p: MaybeArray<string>): FileStatus;
     function read(p: MaybeArray<string>): Buffer;
     function read(p: MaybeArray<string>, encoding: BufferEncoding): string;
     function cp(from: MaybeArray<string>, to: MaybeArray<string>): void;
     function mv(from: MaybeArray<string>, to: MaybeArray<string>): void;
     function ls(p: MaybeArray<string>): string[];
+    /**
+     * check availability to export a file with specified directory and file name.
+     * if it doesn't, retry to check after appending incremental number (e.g. `.1`) to the filename.
+     * @param dir destination directory path.
+     * @param fname file name wanna export to.
+     * @returns exportable file path.
+     */
+    function reserveFilePath(dir: MaybeArray<string>, fname: string): string;
     function joinPath(...p: MaybeArray<string>[]): string;
 }
+export {};

package/dist/func/u-file.js

@@ -45,14 +45,27 @@ var UFile;
         fs.writeFileSync(joinPath(p), c);
     }
     UFile.write = write;
+    /**
+     * remove a file. no error if the file to be removed doesn't exist.
+     */
     function rm(p) {
-        fs.rmSync(joinPath(p), { recursive: true });
+        const pt = joinPath(p);
+        if (fs.existsSync(pt))
+            fs.rmSync(pt, { recursive: true });
     }
     UFile.rm = rm;
     function exists(p) {
         return fs.existsSync(joinPath(p));
     }
     UFile.exists = exists;
+    /**
+     * return a file status. if the file of the status doesn't exist, this returns `null`.
+     */
+    function status(p) {
+        const pt = joinPath(p);
+        return fs.existsSync(pt) ? fs.statSync(pt) : null;
+    }
+    UFile.status = status;
     function read(p, encoding) {
         const f = joinPath(p);
         if (!fs.existsSync(f))
@@ -75,12 +88,31 @@ var UFile;
     }
     UFile.mv = mv;
     function ls(p) {
-        const dir = joinPath(p);
-        if (!fs.statSync(dir).isDirectory())
+        const pt = joinPath(p);
+        if (!pt || !fs.statSync(pt).isDirectory())
             throw new xjs_err_1.XjsErr(s_errCode, "Specified path for ls is not directory.");
-        return fs.readdirSync(dir);
+        return fs.readdirSync(pt);
     }
     UFile.ls = ls;
+    /**
+     * check availability to export a file with specified directory and file name.
+     * if it doesn't, retry to check after appending incremental number (e.g. `.1`) to the filename.
+     * @param dir destination directory path.
+     * @param fname file name wanna export to.
+     * @returns exportable file path.
+     */
+    function reserveFilePath(dir, fname) {
+        const pt = joinPath(dir);
+        if (!pt || !fs.statSync(pt).isDirectory())
+            throw new xjs_err_1.XjsErr(s_errCode, "Specified directory path is not directory.");
+        if (!fname || fname.match(/[\\/:*?"<>|]/))
+            throw new xjs_err_1.XjsErr(s_errCode, "Specified filename is invalid due to empty or including disallowed characters.");
+        let dest = joinPath(pt, fname), i = 1;
+        while (fs.existsSync(dest))
+            dest = joinPath(pt, `${fname}.${i++}`);
+        return dest;
+    }
+    UFile.reserveFilePath = reserveFilePath;
     function joinPath(...p) {
         return path.join(...p.flatMap(u_type_1.UType.takeAsArray));
     }

package/dist/prcs/http/http-resolver-context.d.ts

@@ -1,5 +1,5 @@
 import { ClientMode } from "./http-resolver";
-import { ClientOption, IHttpClient, RequestOption } from "./i-http-client";
+import { ClientOption, HttpResponse, IHttpClient, RequestOption } from "./i-http-client";
 import { Loggable } from "../../const/types";
 export declare const s_clientMode: Record<string, ClientMode>;
 export declare class HttpResolverContext implements IHttpClient {
@@ -16,29 +16,34 @@ export declare class HttpResolverContext implements IHttpClient {
      * request GET to the url.
      * @param url target url.
      * @param op.headers http headers.
-     * @param op.ignoreQuery if true, query part in the `url` is ignored.
+     * @param op.ignoreQuery {@link RequestOption.ignoreQuery}
+     * @param op.downloadPath {@link RequestOption.downloadPath}
      * @returns string encoded by utf-8 as response payload.
      */
     get(url: string, op?: RequestOption & {
         outerRedirectCount?: number;
-    }): Promise<any>;
+    }): Promise<HttpResponse>;
     /**
      * request POST to the url.
      * @param url target url.
      * @param payload request payload. if this is an object, it is treated as json.
      * @param op.headers http headers.
-     * @param op.ignoreQuery if true, query part in the `url` is ignored.
+     * @param op.ignoreQuery {@link RequestOption.ignoreQuery}
+     * @param op.downloadPath {@link RequestOption.downloadPath}
      * @returns string encoded by utf-8 as response payload.
      */
-    post(url: string, payload: any, op?: RequestOption): Promise<any>;
+    post(url: string, payload: any, op?: RequestOption): Promise<HttpResponse>;
     private createProxyAgent;
     private getIn;
     private postIn;
     private reqHttps;
+    private processResponse;
+    private resolveDownloadPath;
     private handleRedirect;
     private createCiphers;
     private setCookies;
     private storeCookies;
     private log;
     private warn;
+    private error;
 }

package/dist/prcs/http/http-resolver-context.js

@@ -26,6 +26,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.HttpResolverContext = exports.s_clientMode = void 0;
 const tls = __importStar(require("tls"));
 const zlib = __importStar(require("zlib"));
+const fs = __importStar(require("fs"));
 const url_1 = require("url");
 const https_1 = require("https");
 const http_1 = require("http");
@@ -35,6 +36,8 @@ const u_http_1 = require("../../func/u-http");
 const u_array_1 = require("../../func/u-array");
 const http_method_1 = require("../../const/http-method");
 const u_type_1 = require("../../func/u-type");
+const u_file_1 = require("../../func/u-file");
+const u_string_1 = require("../../func/u-string");
 exports.s_clientMode = {
     nodejs: { id: 0, cipherOrder: null },
     chrome: { id: 1, cipherOrder: [2, 0, 1] },
@@ -100,7 +103,8 @@ class HttpResolverContext {
      * request GET to the url.
      * @param url target url.
      * @param op.headers http headers.
-     * @param op.ignoreQuery if true, query part in the `url` is ignored.
+     * @param op.ignoreQuery {@link RequestOption.ignoreQuery}
+     * @param op.downloadPath {@link RequestOption.downloadPath}
      * @returns string encoded by utf-8 as response payload.
      */
     async get(url, op) {
@@ -115,7 +119,8 @@ class HttpResolverContext {
      * @param url target url.
      * @param payload request payload. if this is an object, it is treated as json.
      * @param op.headers http headers.
-     * @param op.ignoreQuery if true, query part in the `url` is ignored.
+     * @param op.ignoreQuery {@link RequestOption.ignoreQuery}
+     * @param op.downloadPath {@link RequestOption.downloadPath}
      * @returns string encoded by utf-8 as response payload.
      */
     async post(url, payload, op) {
@@ -185,38 +190,7 @@ class HttpResolverContext {
         if (this._cookies)
             this.setCookies(params.headers);
         return new Promise((resolve, reject) => {
-            const req = (0, https_1.request)(params, (res) => {
-                if (res.headers["set-cookie"])
-                    this.storeCookies(res.headers["set-cookie"]);
-                const sc = u_http_1.UHttp.statusCategoryOf(res.statusCode);
-                if (sc === 3) {
-                    this.handleRedirect(res, params.host).then(resolve).catch(reject).finally(() => res.destroy());
-                    return;
-                }
-                const bfs = [];
-                const contentEncofing = res.headers["content-encoding"]?.toLocaleLowerCase();
-                res.on('data', chunk => bfs.push(chunk));
-                res.on('end', () => {
-                    try {
-                        let retBuf = Buffer.concat(bfs);
-                        if (contentEncofing == "gzip")
-                            retBuf = zlib.gunzipSync(retBuf);
-                        else if (contentEncofing == "br")
-                            retBuf = zlib.brotliDecompressSync(retBuf);
-                        const data = retBuf.toString("utf8");
-                        if (sc !== 2) {
-                            if (data.trim())
-                                this.warn(data);
-                            reject(new xjs_err_1.XjsErr(s_errCode, `Https received a error status ${res.statusCode}`));
-                        }
-                        else
-                            resolve(data);
-                    }
-                    catch (e) {
-                        reject(e);
-                    }
-                });
-            });
+            const req = (0, https_1.request)(params, (res) => this.processResponse(resolve, reject, rc, params.host, res));
             req.on('error', reject);
             req.on('timeout', () => {
                 req.destroy();
@@ -227,6 +201,76 @@ class HttpResolverContext {
             req.end();
         });
     }
+    processResponse(resolve, reject, rc, host, res) {
+        if (res.headers["set-cookie"])
+            this.storeCookies(res.headers["set-cookie"]);
+        const sc = u_http_1.UHttp.statusCategoryOf(res.statusCode);
+        if (sc === 3) {
+            this.handleRedirect(res, host).then(resolve).catch(reject).finally(() => res.destroy());
+            return;
+        }
+        if (res.headers["content-disposition"]?.trim().startsWith("attachment")) {
+            try {
+                const dest = this.resolveDownloadPath(rc.downloadPath, res.headers["content-disposition"]);
+                const stream = fs.createWriteStream(dest);
+                res.pipe(stream);
+                stream.on("finish", () => stream.close());
+                resolve({ headers: res.headers });
+            }
+            catch (e) {
+                this.error(e);
+                reject(new xjs_err_1.XjsErr(s_errCode, "Failed to download a file."));
+            }
+        }
+        const bfs = [];
+        const contentEncofing = res.headers["content-encoding"]?.toLocaleLowerCase();
+        res.on('data', chunk => bfs.push(chunk));
+        res.on('end', () => {
+            try {
+                let retBuf = Buffer.concat(bfs);
+                if (contentEncofing == "gzip")
+                    retBuf = zlib.gunzipSync(retBuf);
+                else if (contentEncofing == "br")
+                    retBuf = zlib.brotliDecompressSync(retBuf);
+                const data = retBuf.toString("utf8");
+                if (sc !== 2) {
+                    if (data.trim())
+                        this.warn(data);
+                    reject(new xjs_err_1.XjsErr(s_errCode, `Https received a error status ${res.statusCode}`));
+                }
+                else
+                    resolve({ payload: data, headers: res.headers });
+            }
+            catch (e) {
+                reject(e);
+            }
+        });
+    }
+    resolveDownloadPath(opPath, disposition) {
+        const appendFname = (d) => {
+            const fname = disposition.split(";")
+                .find(f => f.trim().startsWith("filename"))
+                ?.replace(/^\s+filename\s+=/, "").trim()
+                ?? u_file_1.UFile.reserveFilePath(d, `xjs-download_${u_string_1.UString.simpleTime()}`);
+            return u_file_1.UFile.joinPath(d, fname);
+        };
+        if (opPath) {
+            const st = u_file_1.UFile.status(opPath);
+            if (!st || st.isFile()) {
+                const pathArray = opPath.split("/");
+                pathArray.pop();
+                if (!u_file_1.UFile.exists(pathArray))
+                    throw new xjs_err_1.XjsErr(s_errCode, "Directory of the download path was not found.");
+                return opPath;
+            }
+            if (st.isDirectory()) {
+                if (!u_file_1.UFile.exists(opPath))
+                    throw new xjs_err_1.XjsErr(s_errCode, "Directory of the download path was not found.");
+                return appendFname(opPath);
+            }
+        }
+        return appendFname("./");
+    }
     async handleRedirect(res, host) {
         const rc = this._als.getStore();
         if (!res.headers.location)
@@ -283,5 +327,8 @@ class HttpResolverContext {
     warn(msg) {
         this._l.warn(`[http-resolver] ${msg}`);
     }
+    error(msg) {
+        this._l.error(`[http-resolver] ${msg}`);
+    }
 }
 exports.HttpResolverContext = HttpResolverContext;

package/dist/prcs/http/http-resolver.d.ts

@@ -1,6 +1,6 @@
 import { Loggable } from "../../const/types";
 import { HttpResolverContext } from "./http-resolver-context";
-import { ClientOption, IHttpClient, RequestOption } from "./i-http-client";
+import { ClientOption, HttpResponse, IHttpClient, RequestOption } from "./i-http-client";
 export interface ClientMode {
     id: number;
     cipherOrder: number[];
@@ -20,7 +20,7 @@ export declare class HttpResolver implements IHttpClient {
      * @param _baseCmv chrome major version refered when construct a user agent, and the version will be randomized between `n` to `n-4`.
      * @param _l custom logger. default is `console`.
      */
-    constructor(_baseCmv: number, _l?: Loggable);
+    constructor(_baseCmv?: number, _l?: Loggable);
     /**
      * create a http client as new context that keeps some states. (browser type, cookies, ciphers order, etc...)
      * @param op.mode {@link s_clientMode} that is imitated. default is random between chrome or firefox.
@@ -30,7 +30,7 @@ export declare class HttpResolver implements IHttpClient {
     newContext(op?: ClientOption): HttpResolverContext;
     get(url: string, op?: RequestOption & ClientOption & {
         redirectAsNewRequest?: boolean;
-    }): Promise<any>;
-    post(url: string, payload: any, op?: RequestOption & ClientOption): Promise<any>;
+    }): Promise<HttpResponse>;
+    post(url: string, payload: any, op?: RequestOption & ClientOption): Promise<HttpResponse>;
     private fixCmv;
 }

package/dist/prcs/http/http-resolver.js

@@ -4,6 +4,7 @@ exports.HttpResolver = void 0;
 const xjs_err_1 = require("../../obj/xjs-err");
 const http_resolver_context_1 = require("./http-resolver-context");
 const s_cmvRange = 5;
+const s_defaultCmv = 137;
 class HttpResolver {
     _baseCmv;
     _l;
@@ -11,7 +12,7 @@ class HttpResolver {
      * @param _baseCmv chrome major version refered when construct a user agent, and the version will be randomized between `n` to `n-4`.
      * @param _l custom logger. default is `console`.
      */
-    constructor(_baseCmv, _l = console) {
+    constructor(_baseCmv = s_defaultCmv, _l = console) {
         this._baseCmv = _baseCmv;
         this._l = _l;
     }

package/dist/prcs/http/i-http-client.d.ts

@@ -1,13 +1,31 @@
 /// <reference types="node" />
-import { OutgoingHttpHeaders } from "http";
+import { IncomingHttpHeaders, OutgoingHttpHeaders } from "http";
 import { ClientMode, ProxyConfig } from "./http-resolver";
 export interface ClientOption {
     mode?: ClientMode;
     proxy?: ProxyConfig;
 }
 export interface RequestOption {
-    ignoreQuery?: boolean;
     headers?: OutgoingHttpHeaders;
+    /**
+     * if true, query part in the `url` is ignored.
+     */
+    ignoreQuery?: boolean;
+    /**
+     * destination directory or file path for download. this is only used when `Content-Disposition` header exists.
+     * default is current directory of the process with `filename` of the disposition.
+     */
+    downloadPath?: string;
+}
+export interface HttpResponse {
+    /**
+     * http headers in the response.
+     */
+    headers?: IncomingHttpHeaders;
+    /**
+     * string encoded by utf-8 as response payload.
+     */
+    payload?: string;
 }
 export interface IHttpClient {
     /**
@@ -16,13 +34,14 @@ export interface IHttpClient {
      * @param op.headers http headers.
      * @param op.mode {@link s_clientMode} that is imitated. default is random between chrome or firefox.
      * @param op.proxy proxy configuration.
-     * @param op.ignoreQuery if true, query part in the `url` is ignored.
+     * @param op.ignoreQuery {@link RequestOption.ignoreQuery}
+     * @param op.downloadPath {@link RequestOption.downloadPath}
      * @param op.redirectAsNewRequest handle redirect as new request. this may be efficient when using proxy which is implemented reverse proxy.
-     * @returns string encoded by utf-8 as response payload.
+     * @returns http response. {@link HttpResponse}
      */
     get(url: string, op?: RequestOption & ClientOption & {
         redirectAsNewRequest?: boolean;
-    }): Promise<any>;
+    }): Promise<HttpResponse>;
     /**
      * request POST to the url with new context.
      * @param url target url.
@@ -30,8 +49,9 @@ export interface IHttpClient {
      * @param op.headers http headers.
      * @param op.mode {@link s_clientMode} that is imitated. default is random between chrome or firefox.
      * @param op.proxy proxy configuration.
-     * @param op.ignoreQuery if true, query part in the `url` is ignored.
-     * @returns string encoded by utf-8 as response payload.
+     * @param op.ignoreQuery {@link RequestOption.ignoreQuery}
+     * @param op.downloadPath {@link RequestOption.downloadPath}
+     * @returns http response. {@link HttpResponse}
      */
-    post(url: string, payload: any, op?: RequestOption & ClientOption): Promise<any>;
+    post(url: string, payload: any, op?: RequestOption & ClientOption): Promise<HttpResponse>;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xjs-common",
-  "version": "8.4.1",
+  "version": "9.0.0",
   "description": "library modules for nodejs + typescript that bundled general-purpose implementations.",
   "repository": {
     "type": "git",