utilitas 2000.3.47 → 2000.3.49

package/lib/alan.mjs

@@ -1398,6 +1398,7 @@ const trimText = async (text, limit = Infinity) => {
     text = ensureString(text, { trim: true });
     let trimmed = false;
     let lastCheck = null;
+    limit = Math.max(limit, 0);
     while ((lastCheck = await countTokens(
         buildTextWithEllipsis(text, trimmed), { fast: true }
     )) > limit) {

package/lib/manifest.mjs

@@ -1,7 +1,7 @@
 const manifest = {
     "name": "utilitas",
     "description": "Just another common utility for JavaScript.",
-    "version": "2000.3.47",
+    "version": "2000.3.49",
     "private": false,
     "homepage": "https://github.com/Leask/utilitas",
     "main": "index.mjs",

package/lib/rag.mjs

@@ -1,6 +1,6 @@
-import { convert } from './storage.mjs';
+import { BASE64, convert } from './storage.mjs';
+import { countTokens, trimText } from './alan.mjs';
 import { ensureArray, ensureString, need } from './utilitas.mjs';
-import { trimText } from './alan.mjs';
 
 const _NEED = ['openai', '@google-cloud/discoveryengine'];
 const embeddingClients = {};
@@ -12,18 +12,18 @@ const [
     OPENAI_MODEL_EMBED_SMALL,
     OPENAI_MODEL_EMBED_LARGE,
     GOOGLE_MODEL_GEMINI_EMBED,
-    JINA_MODEL_CLIP_2,
-    JINA_MODEL_V_3,
+    JINA_MODEL_V_4,
     GOOGLE_MODEL_SEMANTIC_RANKER,
+    JINA_MODEL_RERANKER_M0,
 ] = [
         'OPENAI', 'GOOGLE', 'OPENROUTER', 'JINA',
         'global', 'default_ranking_config',
         'text-embedding-3-small', // dim: 1536
         'text-embedding-3-large', // dim: 3072
         'gemini-embedding-001', // dim: 768(default), 1536, or 3072(google default)
-        'jina-clip-v2', // dim: 1024
-        'jina-embeddings-v3', // dim: 256‑1024
+        'jina-embeddings-v4', // dim: 256‑2048
         'semantic-ranker-default@latest',
+        'jina-reranker-m0',
     ];
 
 const PROVIDER_BASE_URL = {
@@ -34,11 +34,12 @@ const PROVIDER_BASE_URL = {
 const DEFAULT_EMBEDDING_MODELS = {
     [OPENAI]: OPENAI_MODEL_EMBED_SMALL,
     [OPENROUTER]: GOOGLE_MODEL_GEMINI_EMBED,
-    [JINA]: JINA_MODEL_CLIP_2,
+    [JINA]: JINA_MODEL_V_4,
 };
 
 const DEFAULT_RERANKER_MODELS = {
     [GOOGLE]: GOOGLE_MODEL_SEMANTIC_RANKER,
+    [JINA]: JINA_MODEL_RERANKER_M0,
 };
 
 const MODEL_CONFIG = {
@@ -52,29 +53,25 @@ const MODEL_CONFIG = {
         source: 'google', image: false, maxTokens: 2048,
         options: { dimensions: 768 },
     },
-    [JINA_MODEL_CLIP_2]: {
-        source: 'jina', image: true, maxTokens: 8192, options: {
-            task: 'retrieval.query', dimensions: 1024,
-            normalized: true, embedding_type: 'float',
-        },
-    },
     // Token calculation may be incorrect because its limitation applies to the
     // entire request rather than individual entries.
-    [JINA_MODEL_V_3]: {
-        source: 'jina', image: false, maxTokens: 8192, options: {
-            task: 'retrieval.query', dimensions: 1024, normalized: true,
-            late_chunking: true, embedding_type: 'float',
+    // https://jina.ai/embeddings
+    [JINA_MODEL_V_4]: {
+        source: 'jina', image: true, maxTokens: 8192, recordsLimit: 512,
+        options: {
+            task: 'text-matching', // 'retrieval.query', 'retrieval.passage'
+            dimensions: 768, // normalized: true, by default DONT submit
+            truncate: true, // late_chunking: true, by default DONT submit
+            embedding_type: 'float',
         },
     },
     [GOOGLE_MODEL_SEMANTIC_RANKER]: {
-        source: 'google', image: false, maxTokens: 8192,
-        options: {
-            // task: 'retrieval.query',
-            // dimensions: 1024,
-            // normalized: true,
-            // late_chunking: true,
-            // embedding_type: 'float',
-        },
+        source: 'google', image: false, maxTokens: 1024, recordsLimit: 200,
+        options: { ignoreRecordDetailsInResponse: true },
+    },
+    [JINA_MODEL_RERANKER_M0]: {
+        source: 'jina', image: true, maxTokens: 1024, recordsLimit: 200,
+        options: { return_documents: false },
     },
 };
 
@@ -122,6 +119,10 @@ const getRerankerClient = (provider) => {
 };
 
 const initEmbedding = async (options = {}) => {
+    if (options?.debug) {
+        (await need('node:util')).inspect.defaultOptions.depth = null;
+        options.logLevel = 'debug';
+    }
     ensureApiKey(options);
     const provider = ensureEmbeddingProvider(options);
     const OpenAI = await need('openai');
@@ -154,7 +155,7 @@ const embed = async (input, options = {}) => {
             );
             if (options?.input) {
                 x.image = await convert(
-                    x.image, { ...options, expected: 'base64' }
+                    x.image, { ...options, expected: BASE64 }
                 );
             }
         }
@@ -188,6 +189,7 @@ const embed = async (input, options = {}) => {
 
 const initReranker = async (options = {}) => {
     const provider = ensureRerankerProvider(options);
+    const model = options?.model || DEFAULT_RERANKER_MODELS[provider];
     switch (provider) {
         case GOOGLE:
             ensureGoogleCredentials(options);
@@ -202,40 +204,81 @@ const initReranker = async (options = {}) => {
             };
             const client = new RankServiceClient(clientOptions);
             rerankerClients[provider] = {
-                model: options?.model || DEFAULT_RERANKER_MODELS[provider],
-                client, rankingConfigPath: client.rankingConfigPath(
+                client, model, rankingConfigPath: client.rankingConfigPath(
                     options.projectId, location,
                     options?.rerankerConfigId || GOOGLE_RERANK_CONFIG_ID
                 ),
             };
             break;
+        case JINA:
+            const OpenAI = await need('openai');
+            const baseURL = options?.baseURL || PROVIDER_BASE_URL[provider];
+            rerankerClients[provider] = {
+                client: new OpenAI({ ...options, baseURL }),
+                model, source: MODEL_CONFIG[model]?.source,
+            };
+            break;
         default:
             throw new Error(`Unsupported reranker provider: ${provider}`);
     }
     return getRerankerClient(provider);
 };
 
-const rerank = async (query, documents, options = {}) => {
+const rerank = async (query, records, options = {}) => {
     assert(query, 'Query is required.', 400);
-    assert(documents?.length, 'Documents are required.', 400);
-    const records = documents.map((doc, idx) => ({
-        id: String(idx), content: doc,
-    }));
+    assert(records?.length, 'Records are required.', 400);
     const { provider, model, client, rankingConfigPath }
         = getRerankerClient(options?.provider);
+    records = records.map((content, id) => Object.isObject(content)
+        ? content : { id: String(id), content }).slice(
+            0, MODEL_CONFIG[model]?.recordsLimit || records.length
+        );
+    const maxTokens = MODEL_CONFIG[model]?.maxTokens || Infinity;
     let result;
+    for (let i in records) {
+        records[i].title = await trimText(records[i]?.title || '', maxTokens);
+        const titleTokens = await countTokens(records[i].title);
+        const availableTokens = maxTokens - titleTokens;
+        records[i].content = availableTokens > 0 ? await trimText(
+            records[i].content, availableTokens
+        ) : '';
+        records[i].image = records[i].image ? await convert(records[i].image, {
+            ...options, expected: BASE64,
+        }) : undefined;
+    }
     switch (provider) {
         case GOOGLE:
-            const request = {
+            var body = {
                 model, query, rankingConfig: rankingConfigPath,
-                records, topN: ~~options?.topN || documents.length,
+                records, topN: ~~options?.topN || records.length,
+                ...MODEL_CONFIG[model]?.options || {},
+                ...options?.requestOptions || {},
+            };
+            result = (await client.rank(body))?.[0]?.records;
+            options?.raw || (result = result.map(x => ({
+                index: ~~x.id, score: x.score,
+            })));
+            break;
+        case JINA:
+            records = records.map(x =>
+                ((x.title || x.content) ? { text: [x.title, x.content].filter(x => x).join('\n') } : null)
+                || (x.image ? { image: x.image } : null)
+            ).filter(x => x);
+            assert(records.length, 'No valid records found.', 400);
+            var body = {
+                model, query, documents: records,
+                ...MODEL_CONFIG[model]?.options || {},
+                ...options?.requestOptions || {},
             };
-            result = (await client.rank(request))?.[0]?.records;
+            result = (await client.post('/rerank', { body }))?.results;
+            options?.raw || (result = result.map(x => ({
+                index: ~~(x.index), score: x.relevance_score,
+            })));
             break;
         default:
             throw new Error(`Unsupported reranker provider: ${provider}`);
     }
-    // print(result);
+    result.sort((a, b) => b.score - a.score);
     return result || [];
 };
 

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "utilitas",
     "description": "Just another common utility for JavaScript.",
-    "version": "2000.3.47",
+    "version": "2000.3.49",
     "private": false,
     "homepage": "https://github.com/Leask/utilitas",
     "main": "index.mjs",