utilitas 2000.3.48 → 2000.3.52

package/lib/alan.mjs

@@ -1220,7 +1220,7 @@ const initChat = async (options = {}) => {
             'Invalid session storage provider.'
         );
         chatConfig.sessions = options.sessions;
-    }
+    } else { log(`WARNING: Sessions persistence is not enabled.`); }
     options.instructions && (chatConfig.systemPrompt = options.instructions);
     // Use Gemini instead of ChatGPT because of the longer package.
     const [spTokens, ais] = await Promise.all([countTokens([await buildMessage(
@@ -1446,6 +1446,7 @@ export {
     prompt,
     promptOpenRouter,
     resetSession,
+    setSession,
     stt,
     talk,
     trimPrompt,

package/lib/manifest.mjs

@@ -1,7 +1,7 @@
 const manifest = {
     "name": "utilitas",
     "description": "Just another common utility for JavaScript.",
-    "version": "2000.3.48",
+    "version": "2000.3.52",
     "private": false,
     "homepage": "https://github.com/Leask/utilitas",
     "main": "index.mjs",

package/lib/rag.mjs

@@ -1,5 +1,5 @@
+import { BASE64, convert } from './storage.mjs';
 import { countTokens, trimText } from './alan.mjs';
-import { convert } from './storage.mjs';
 import { ensureArray, ensureString, need } from './utilitas.mjs';
 
 const _NEED = ['openai', '@google-cloud/discoveryengine'];
@@ -14,6 +14,7 @@ const [
     GOOGLE_MODEL_GEMINI_EMBED,
     JINA_MODEL_V_4,
     GOOGLE_MODEL_SEMANTIC_RANKER,
+    JINA_MODEL_RERANKER_M0,
 ] = [
         'OPENAI', 'GOOGLE', 'OPENROUTER', 'JINA',
         'global', 'default_ranking_config',
@@ -22,6 +23,7 @@ const [
         'gemini-embedding-001', // dim: 768(default), 1536, or 3072(google default)
         'jina-embeddings-v4', // dim: 256‑2048
         'semantic-ranker-default@latest',
+        'jina-reranker-m0',
     ];
 
 const PROVIDER_BASE_URL = {
@@ -37,6 +39,7 @@ const DEFAULT_EMBEDDING_MODELS = {
 
 const DEFAULT_RERANKER_MODELS = {
     [GOOGLE]: GOOGLE_MODEL_SEMANTIC_RANKER,
+    [JINA]: JINA_MODEL_RERANKER_M0,
 };
 
 const MODEL_CONFIG = {
@@ -64,6 +67,11 @@ const MODEL_CONFIG = {
     },
     [GOOGLE_MODEL_SEMANTIC_RANKER]: {
         source: 'google', image: false, maxTokens: 1024, recordsLimit: 200,
+        options: { ignoreRecordDetailsInResponse: true },
+    },
+    [JINA_MODEL_RERANKER_M0]: {
+        source: 'jina', image: true, maxTokens: 1024, recordsLimit: 200,
+        options: { return_documents: false },
     },
 };
 
@@ -147,7 +155,7 @@ const embed = async (input, options = {}) => {
             );
             if (options?.input) {
                 x.image = await convert(
-                    x.image, { ...options, expected: 'base64' }
+                    x.image, { ...options, expected: BASE64 }
                 );
             }
         }
@@ -181,6 +189,7 @@ const embed = async (input, options = {}) => {
 
 const initReranker = async (options = {}) => {
     const provider = ensureRerankerProvider(options);
+    const model = options?.model || DEFAULT_RERANKER_MODELS[provider];
     switch (provider) {
         case GOOGLE:
             ensureGoogleCredentials(options);
@@ -195,13 +204,20 @@ const initReranker = async (options = {}) => {
             };
             const client = new RankServiceClient(clientOptions);
             rerankerClients[provider] = {
-                model: options?.model || DEFAULT_RERANKER_MODELS[provider],
-                client, rankingConfigPath: client.rankingConfigPath(
+                client, model, rankingConfigPath: client.rankingConfigPath(
                     options.projectId, location,
                     options?.rerankerConfigId || GOOGLE_RERANK_CONFIG_ID
                 ),
             };
             break;
+        case JINA:
+            const OpenAI = await need('openai');
+            const baseURL = options?.baseURL || PROVIDER_BASE_URL[provider];
+            rerankerClients[provider] = {
+                client: new OpenAI({ ...options, baseURL }),
+                model, source: MODEL_CONFIG[model]?.source,
+            };
+            break;
         default:
             throw new Error(`Unsupported reranker provider: ${provider}`);
     }
@@ -226,20 +242,43 @@ const rerank = async (query, records, options = {}) => {
         records[i].content = availableTokens > 0 ? await trimText(
             records[i].content, availableTokens
         ) : '';
+        records[i].image = records[i].image ? await convert(records[i].image, {
+            ...options, expected: BASE64,
+        }) : undefined;
     }
     switch (provider) {
         case GOOGLE:
-            const request = {
+            var body = {
                 model, query, rankingConfig: rankingConfigPath,
                 records, topN: ~~options?.topN || records.length,
+                ...MODEL_CONFIG[model]?.options || {},
+                ...options?.requestOptions || {},
+            };
+            result = (await client.rank(body))?.[0]?.records;
+            options?.raw || (result = result.map(x => ({
+                index: ~~x.id, score: x.score,
+            })));
+            break;
+        case JINA:
+            records = records.map(x =>
+                ((x.title || x.content) ? { text: [x.title, x.content].filter(x => x).join('\n') } : null)
+                || (x.image ? { image: x.image } : null)
+            ).filter(x => x);
+            assert(records.length, 'No valid records found.', 400);
+            var body = {
+                model, query, documents: records,
+                ...MODEL_CONFIG[model]?.options || {},
                 ...options?.requestOptions || {},
             };
-            result = (await client.rank(request))?.[0]?.records;
+            result = (await client.post('/rerank', { body }))?.results;
+            options?.raw || (result = result.map(x => ({
+                index: ~~(x.index), score: x.relevance_score,
+            })));
             break;
         default:
             throw new Error(`Unsupported reranker provider: ${provider}`);
     }
-    // print(result);
+    result.sort((a, b) => b.score - a.score);
     return result || [];
 };
 

package/lib/storage.mjs

@@ -119,18 +119,20 @@ const assertPath = async (path, type, mode, msg, code, options) => {
 };
 
 const isTextFile = async (file, options) => {
-    const isBuf = Buffer.isBuffer(file);
-    let [f, bytes, res] = [isBuf ? file : await fs.open(file, 'r'), null, true];
-    for (let i = 0; i < (~~options?.length || 1000); i++) {
-        let buf = Buffer.alloc(1);
-        if (isBuf) { buf[0] = f[i]; bytes = i < f.length ? 1 : 0; }
-        else { bytes = readSync(f.fd, buf, 0, 1, i); }
-        if (bytes === 0) { break; } else if (
-            bytes === 1 && buf.toString().charCodeAt() === 0
-        ) { res = false; break; }
+    const maxBytes = options?.length ?? 1000;
+    let buf;
+    if (Buffer.isBuffer(file)) {
+        buf = file.subarray(0, maxBytes);
+    } else {
+        const fd = await fs.promises.open(file, 'r');
+        buf = Buffer.alloc(maxBytes);
+        await fd.read(buf, 0, maxBytes, 0);
+        await fd.close();
+    }
+    for (let i = 0; i < buf.length; i++) {
+        if (buf[i] === 0) return false;
     }
-    isBuf || f?.close()
-    return res;
+    return true;
 };
 
 const getConfigFilename = async (options) => {

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "utilitas",
     "description": "Just another common utility for JavaScript.",
-    "version": "2000.3.48",
+    "version": "2000.3.52",
     "private": false,
     "homepage": "https://github.com/Leask/utilitas",
     "main": "index.mjs",