utilitas 2000.3.16 → 2000.3.17

package/lib/manifest.mjs

@@ -1,7 +1,7 @@
 const manifest = {
     "name": "utilitas",
     "description": "Just another common utility for JavaScript.",
-    "version": "2000.3.16",
+    "version": "2000.3.17",
     "private": false,
     "homepage": "https://github.com/Leask/utilitas",
     "main": "index.mjs",
@@ -51,6 +51,7 @@ const manifest = {
         "office-text-extractor": "^3.0.3",
         "openai": "^6.9.1",
         "pdfjs-dist": "^5.4.394",
+        "pdf-lib": "^1.17.1",
         "pg": "^8.16.3",
         "pgvector": "^0.2.1",
         "ping": "^1.0.0",

package/lib/storage.mjs

@@ -1,7 +1,7 @@
 import {
     log as _log,
     base64Decode, base64Encode, ensureString, extract, ignoreErrFunc,
-    mergeAtoB, need, throwError, trim, voidFunc, which,
+    mergeAtoB, need, throwError, trim, which,
 } from './utilitas.mjs';
 
 import { fileTypeFromBuffer } from 'file-type';

package/lib/vision.mjs

@@ -9,7 +9,7 @@ import {
 import { convert, DATAURL, BUFFER, FILE } from './storage.mjs';
 import fs from 'node:fs';
 
-const _NEED = ['office-text-extractor', 'pdfjs-dist', 'tesseract.js'];
+const _NEED = ['office-text-extractor', 'pdfjs-dist', 'pdf-lib', 'tesseract.js'];
 const clients = {};
 const errorMessage = 'Invalid input data.';
 const log = content => _log(content, import.meta.url);
@@ -144,27 +144,96 @@ const ocr = async (file, options = {}) => {
     const client = clients?.[provider];
     assert(client, 'No available OCR provider.');
     const model = options?.model || client.model;
-    const document_url = await convert(file, { ...options, expected: DATAURL });
     switch (provider) {
         case GOOGLE_MISTRAL:
             const key = await getGoogleAuthTokenByAuth(client.auth);
-            return await (await fetch(
-                `https://${client.region}-aiplatform.googleapis.com/v1/`
-                + `projects/${client.project}/locations/${client.region}/`
-                + `publishers/mistralai/models/${model}:rawPredict`, {
-                method: 'POST', headers: {
-                    'Content-Type': 'application/json',
-                    'Authorization': `Bearer ${key}`
-                }, body: JSON.stringify({
-                    model, include_image_base64: true,
-                    document: { type: 'document_url', document_url },
-                })
-            })).json();
+            const inputPdfs = await splitPdf(file, {
+                ...options, expected: DATAURL, size: 2,
+            });
+            const resps = (await Promise.all(inputPdfs.map(
+                async document_url => await (await fetch(
+                    `https://${client.region}-aiplatform.googleapis.com/v1/`
+                    + `projects/${client.project}/locations/${client.region}/`
+                    + `publishers/mistralai/models/${model}:rawPredict`, {
+                    method: 'POST', headers: {
+                        'Content-Type': 'application/json',
+                        'Authorization': `Bearer ${key}`
+                    }, body: JSON.stringify({
+                        model, include_image_base64: true,
+                        document: { type: 'document_url', document_url },
+                    })
+                })).json()
+            ))).filter(x => x?.pages?.length);
+            const resp = {
+                pages: [], usage_info: { pages_processed: 0, doc_size_bytes: 0 }
+            };
+            resps.map(x => {
+                x.pages.map(p => {
+                    p.index = resp.pages.length;
+                    resp.pages.push(p);
+                    p.images.map(i => {
+                        const oId = i.id;
+                        i.id = `page-${p.index}-${oId}`;
+                        p.markdown = p.markdown.replaceAll(
+                            `![${oId}](${oId})`, `![${i.id}](${i.id})`
+                        );
+                    });
+                });
+                resp.model = x.model;
+                resp.usage_info.pages_processed += x.usage_info.pages_processed;
+                resp.usage_info.doc_size_bytes += x.usage_info.doc_size_bytes;
+            });
+            if (options?.raw) { return resp; }
+            else if (options?.paging) { return resp.pages; }
+            const markdown = [];
+            resp.images = {};
+            for (const p of resp.pages) {
+                markdown.push(p.markdown);
+                await Promise.all(p.images.map(async i => {
+                    const id = i.id;
+                    i.width = i.bottom_right_x - i.top_left_x;
+                    i.height = i.bottom_right_y - i.top_left_y;
+                    i.annotation = i.image_annotation;
+                    i.data = await convert(i.image_base64, {
+                        ...options, input: 'DATAURL',
+                    });
+                    [
+                        'id', 'image_annotation', 'image_base64', 'top_left_x',
+                        'top_left_y', 'bottom_right_x', 'bottom_right_y',
+                    ].map(k => delete i[k]);
+                    resp.images[id] = i;
+                }));
+            }
+            resp.text = markdown.join('\n\n');
+            delete resp.pages;
+            return resp;
         default:
             throw new Error('Invalid provider.');
     }
 };
 
+const splitPdf = async (file, options) => {
+    const [content, { PDFDocument }] = await Promise.all([
+        convert(file, { ...options, expected: BUFFER }), need('pdf-lib')
+    ]);
+    const [doc, result] = [await PDFDocument.load(content), []];
+    const count = doc.getPageCount();
+    const size = ~~options?.size || Infinity;
+    for (let i = 0; i < count; i += size) {
+        result.push((async () => {
+            const sub = await PDFDocument.create();
+            const copied = await sub.copyPages(doc, Array.from(
+                { length: Math.min(size, count - i) }, (_, j) => i + j
+            ));
+            copied.forEach(page => sub.addPage(page));
+            return await convert(Buffer.from(await sub.save()), {
+                ...options, input: 'BUFFER',
+            });
+        })());
+    }
+    return await Promise.all(result);
+};
+
 export default init;
 export {
     _NEED,
@@ -174,4 +243,5 @@ export {
     ocr,
     ocrImage,
     parseOfficeFile,
+    splitPdf,
 };

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "utilitas",
     "description": "Just another common utility for JavaScript.",
-    "version": "2000.3.16",
+    "version": "2000.3.17",
     "private": false,
     "homepage": "https://github.com/Leask/utilitas",
     "main": "index.mjs",
@@ -62,6 +62,7 @@
         "office-text-extractor": "^3.0.3",
         "openai": "^6.9.1",
         "pdfjs-dist": "^5.4.394",
+        "pdf-lib": "^1.17.1",
         "pg": "^8.16.3",
         "pgvector": "^0.2.1",
         "ping": "^1.0.0",