utilitas 1995.2.2 → 1995.2.4

package/lib/manifest.mjs

@@ -1,7 +1,7 @@
 const manifest = {
     "name": "utilitas",
     "description": "Just another common utility for JavaScript.",
-    "version": "1995.2.2",
+    "version": "1995.2.4",
     "private": false,
     "homepage": "https://github.com/Leask/utilitas",
     "main": "index.mjs",

package/lib/storage.mjs

@@ -16,12 +16,14 @@ const _NEED = ['mime-types', '@google-cloud/storage'];
 const errorMessage = 'Invalid file.';
 const defaultMetadata = { cacheControl: 'public, max-age=31536000' };
 const getGcUrlByBucket = bucket => `https://storage.cloud.google.com/${bucket}`;
+const getGsById = id => `gs://${id}`.replace(/[\/][^\/]*$/, '');
+const getIdByGs = gs => gs.replace(/^gs:\/\/[^/]*\/(.*)$/, '$1');
 const mapFilename = name => join(name.substr(0, 2), name.substr(2, 2));
 const [_zip, _unzip] = [__zip, __unzip].map(promisify);
 const log = content => _log(content, import.meta.url);
 
-const [NULL, BASE64, BUFFER, FILE, STREAM, encoding, BINARY, mode, dirMode]
-    = ['NULL', 'BASE64', 'BUFFER', 'FILE', 'STREAM', 'utf8', 'binary', '0644', '0755'];
+const [NULL, BASE64, BUFFER, FILE, STREAM, TEXT, _JSON, encoding, BINARY, mode, dirMode]
+    = ['NULL', 'BASE64', 'BUFFER', 'FILE', 'STREAM', 'TEXT', 'JSON', 'utf8', 'binary', '0644', '0755'];
 
 const [encodeBase64, encodeBinary, encodeNull]
     = [{ encoding: BASE64 }, { encoding: BINARY }, { encoding: NULL }];
@@ -176,6 +178,7 @@ const outputFile = async (buffer, options) => {
         return options?.file;
     }
     return await writeTempFile(buffer, {
+        filename: options?.filename,
         encoding: _encoding, suffix: options?.suffix,
     });
 };
@@ -200,8 +203,16 @@ const convert = async (any, options) => {
         (Buffer.isBuffer(any) || ArrayBuffer.isArrayBuffer(any)) ? BUFFER : options?.input,
         options?.expected || BUFFER
     ].map(x => ensureString(x, { case: 'UP' }));
-    const [oriFile, tarStm] = [input === FILE ? any : null, expected === STREAM];
-    let [meta, subExp] = [null, tarStm ? FILE : expected];
+    let [oriFile, meta, subExp] =
+        [input === FILE ? any : null, null, expected];
+    switch (expected) {
+        case STREAM:
+            subExp = FILE;
+            break;
+        case TEXT:
+        case _JSON:
+            subExp = BUFFER;
+    }
     oriFile && (meta = await assertPath(any, 'F', 'R'));
     if (input !== subExp) {
         switch (`${input}-${subExp}`) {
@@ -214,7 +225,18 @@ const convert = async (any, options) => {
             default: throwError('Invalid input or expected format.', 400);
         }
     }
-    tarStm && (any = createReadStream(any));
+    switch (expected) {
+        case STREAM:
+            any = createReadStream(any);
+            break;
+        case TEXT:
+        case _JSON:
+            any = any.toString();
+    }
+    switch (expected) {
+        case _JSON:
+            any = JSON.parse(any);
+    }
     oriFile && subExp !== FILE && options?.cleanup && await tryRm(ori, options);
     return options?.meta ? { meta, content: any } : any;
 };
@@ -322,16 +344,74 @@ const uploadToCloud = async (data, options) => {
         suffix: options?.suffix, ...options || {},
     });
     const raw = await client.upload(file, {
-        gzip: true, destination: options?.destination, metadata: defaultMetadata
+        gzip: true, destination: options?.destination || join(
+            ...options?.prefix ? [options.prefix] : [], basename(file)
+        ), metadata: defaultMetadata, ...options || {},
     });
     const result = options?.raw ? raw : raw[0].metadata;
     input !== FILE && await tryRm(file.path);
+    !options?.raw && result && (result.gs = getGsById(result?.id));
+    return result;
+};
+
+const downloadFileFromCloud = async (path, options) => {
+    assert(client, 'Cloud storage has not been initialized.', 500);
+    let result = await client.file(path).download(options);
+    if (options?.raw) { return result; }
+    result = await convert(
+        result[0], { ...options || {}, input: BUFFER }
+    );
+    if (options?.text) {
+        result = result.toString();
+    }
+    return result;
+};
+
+const downloadFromCloud = async (path, options) => {
+    assert(path, 'Path is required.', 400);
+    const isFolder = path.endsWith('/');
+    const paths = isFolder ? await lsOnCloud(path, { name: true }) : [path];
+    const resp = await Promise.all(paths.map(
+        x => downloadFileFromCloud(x, options)
+    ));
+    if (!isFolder) { return resp[0]; }
+    const result = {};
+    for (let i in paths) { result[paths[i]] = resp[i]; }
     return result;
 };
 
-const existsOnCloud = async (destination) => {
+const existsOnCloud = async (destination, options) => {
+    assert(client, 'Cloud storage has not been initialized.', 500);
+    const result = await client.file(destination).exists();
+    return options?.raw ? result : (result[0] ? {} : null);
+};
+
+const lsOnCloud = async (prefix, options) => {
     assert(client, 'Cloud storage has not been initialized.', 500);
-    return (await client.file(destination).exists())[0] ? {} : null;
+    let result = await client.getFiles({ prefix, ...options || {} });
+    if (options?.raw) { return result; }
+    result = (result[0] || []).map(x => x.metadata);
+    if (options?.name) { result = result.map(x => x.name); }
+    return result;
+};
+
+const deleteFileOnCloud = async (path, options) => {
+    assert(client, 'Cloud storage has not been initialized.', 500);
+    const result = await client.file(path).delete(options);
+    return options?.raw ? result : result[0].toJSON();
+};
+
+const deleteOnCloud = async (path, options) => {
+    assert(path, 'Path is required.', 400);
+    const isFolder = path.endsWith('/');
+    const paths = isFolder ? await lsOnCloud(path, { name: true }) : [path];
+    const resp = await Promise.all(paths.map(
+        x => deleteFileOnCloud(x, options)
+    ));
+    if (!isFolder) { return resp[0]; }
+    const result = {};
+    for (let i in paths) { result[paths[i]] = resp[i]; }
+    return result;
 };
 
 export {
@@ -339,17 +419,23 @@ export {
     analyzeFile,
     assertPath,
     convert,
+    deleteFileOnCloud,
+    deleteOnCloud,
+    downloadFileFromCloud,
+    downloadFromCloud,
     encodeBase64DataURL,
     exists,
     existsOnCloud,
     getConfig,
     getConfigFilename,
     getGcUrlByBucket,
+    getIdByGs,
     getTempPath,
     handleError,
     init,
     isTextFile,
     legalFilename,
+    lsOnCloud,
     mapFilename,
     mergeFile,
     readFile,

package/lib/vision.mjs

@@ -1,8 +1,11 @@
-import { convert } from './storage.mjs';
 import { getApiKeyCredentials } from './encryption.mjs';
 
 import {
-    ensureArray, ignoreErrFunc, log as _log, need, throwError, trim,
+    convert, deleteOnCloud, downloadFromCloud, getIdByGs, uploadToCloud,
+} from './storage.mjs';
+
+import {
+    ensureArray, ignoreErrFunc, log as _log, need, throwError, trim, timeout,
 } from './utilitas.mjs';
 
 const _NEED = ['@google-cloud/vision', 'tesseract.js'];
@@ -15,10 +18,11 @@ let client;
 
 const init = async (options) => {
     if (options) {
-        if (options?.apiKey) {
-            const sslCreds = await getApiKeyCredentials(options);
+        if (options?.credentials || options?.apiKey) {
             const vision = (await need('@google-cloud/vision')).default;
-            client = new vision.ImageAnnotatorClient({ sslCreds });
+            client = new vision.ImageAnnotatorClient(options?.apiKey ? {
+                sslCreds: await getApiKeyCredentials(options)
+            } : options);
         } else { await checkTesseract({ assert: true }); }
     }
     assert(
@@ -133,22 +137,36 @@ const read = async (image, options) => {
     return pages;
 };
 
-const read2 = async (image, options) => {
-    // assert(client, 'Vision API has not been initialized.', 500);
-    // const content = await convert(image, {
-    //     input: options?.input, expected: BASE64, errorMessage,
-    // });
-
-    // const [response] = await client.asyncBatchAnnotateFiles({
-    //     requests: [{
-    //         inputConfig: { mimeType: 'application/pdf', content },
-    //         features: [{ type: 'DOCUMENT_TEXT_DETECTION' }],
-    //         pages: [1, 2, 3, 4, 5], // max 5 pages
-    //     }],
-    // });
-    // let pages = response.responses[0].responses;
-    // options?.raw || (pages = pages.map(x => x.fullTextAnnotation.text));
-    // return pages;
+const readAll = async (image, options) => {
+    assert(client, 'Vision API has not been initialized.', 500);
+    const result = {};
+    result.upload = await uploadToCloud(image, {
+        gzip: false, prefix: options?.prefix || '_vision', ...options || {},
+    });
+    const uri = result.upload?.gs;
+    const destination = `${uri}_result/`;
+    const resultId = getIdByGs(destination);
+    result.clear = await deleteOnCloud(resultId);
+    result.submit = await client.asyncBatchAnnotateFiles({
+        requests: [{
+            inputConfig: { mimeType: 'application/pdf', gcsSource: { uri } },
+            outputConfig: { gcsDestination: { uri: destination } },
+            features: [{ type: 'DOCUMENT_TEXT_DETECTION' }],
+        }],
+    });
+    result.response = await result.submit[0].promise();
+    result.result = await downloadFromCloud(resultId, { expected: 'JSON' });
+    options?.keep || (result.cleanup = await Promise.all(
+        [getIdByGs(uri), resultId].map(deleteOnCloud)
+    ));
+    if (options?.raw) { return result; }
+    const pages = [];
+    for (let file in result.result) {
+        result.result[file].responses.map(x => pages.push(
+            x.fullTextAnnotation.text
+        ));
+    }
+    return pages;
 };
 
 export {
@@ -159,6 +177,6 @@ export {
     ocrImageGoogle,
     ocrImageTesseract,
     read,
-    read2,
+    readAll,
     see,
 };

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "utilitas",
     "description": "Just another common utility for JavaScript.",
-    "version": "1995.2.2",
+    "version": "1995.2.4",
     "private": false,
     "homepage": "https://github.com/Leask/utilitas",
     "main": "index.mjs",