utilitas 2000.3.47 → 2000.3.48

package/lib/alan.mjs

@@ -1398,6 +1398,7 @@ const trimText = async (text, limit = Infinity) => {
     text = ensureString(text, { trim: true });
     let trimmed = false;
     let lastCheck = null;
+    limit = Math.max(limit, 0);
     while ((lastCheck = await countTokens(
         buildTextWithEllipsis(text, trimmed), { fast: true }
     )) > limit) {

package/lib/manifest.mjs

@@ -1,7 +1,7 @@
 const manifest = {
     "name": "utilitas",
     "description": "Just another common utility for JavaScript.",
-    "version": "2000.3.47",
+    "version": "2000.3.48",
     "private": false,
     "homepage": "https://github.com/Leask/utilitas",
     "main": "index.mjs",

package/lib/rag.mjs

@@ -1,6 +1,6 @@
+import { countTokens, trimText } from './alan.mjs';
 import { convert } from './storage.mjs';
 import { ensureArray, ensureString, need } from './utilitas.mjs';
-import { trimText } from './alan.mjs';
 
 const _NEED = ['openai', '@google-cloud/discoveryengine'];
 const embeddingClients = {};
@@ -12,8 +12,7 @@ const [
     OPENAI_MODEL_EMBED_SMALL,
     OPENAI_MODEL_EMBED_LARGE,
     GOOGLE_MODEL_GEMINI_EMBED,
-    JINA_MODEL_CLIP_2,
-    JINA_MODEL_V_3,
+    JINA_MODEL_V_4,
     GOOGLE_MODEL_SEMANTIC_RANKER,
 ] = [
         'OPENAI', 'GOOGLE', 'OPENROUTER', 'JINA',
@@ -21,8 +20,7 @@ const [
         'text-embedding-3-small', // dim: 1536
         'text-embedding-3-large', // dim: 3072
         'gemini-embedding-001', // dim: 768(default), 1536, or 3072(google default)
-        'jina-clip-v2', // dim: 1024
-        'jina-embeddings-v3', // dim: 256‑1024
+        'jina-embeddings-v4', // dim: 256‑2048
         'semantic-ranker-default@latest',
     ];
 
@@ -34,7 +32,7 @@ const PROVIDER_BASE_URL = {
 const DEFAULT_EMBEDDING_MODELS = {
     [OPENAI]: OPENAI_MODEL_EMBED_SMALL,
     [OPENROUTER]: GOOGLE_MODEL_GEMINI_EMBED,
-    [JINA]: JINA_MODEL_CLIP_2,
+    [JINA]: JINA_MODEL_V_4,
 };
 
 const DEFAULT_RERANKER_MODELS = {
@@ -52,29 +50,20 @@ const MODEL_CONFIG = {
         source: 'google', image: false, maxTokens: 2048,
         options: { dimensions: 768 },
     },
-    [JINA_MODEL_CLIP_2]: {
-        source: 'jina', image: true, maxTokens: 8192, options: {
-            task: 'retrieval.query', dimensions: 1024,
-            normalized: true, embedding_type: 'float',
-        },
-    },
     // Token calculation may be incorrect because its limitation applies to the
     // entire request rather than individual entries.
-    [JINA_MODEL_V_3]: {
-        source: 'jina', image: false, maxTokens: 8192, options: {
-            task: 'retrieval.query', dimensions: 1024, normalized: true,
-            late_chunking: true, embedding_type: 'float',
+    // https://jina.ai/embeddings
+    [JINA_MODEL_V_4]: {
+        source: 'jina', image: true, maxTokens: 8192, recordsLimit: 512,
+        options: {
+            task: 'text-matching', // 'retrieval.query', 'retrieval.passage'
+            dimensions: 768, // normalized: true, by default DONT submit
+            truncate: true, // late_chunking: true, by default DONT submit
+            embedding_type: 'float',
         },
     },
     [GOOGLE_MODEL_SEMANTIC_RANKER]: {
-        source: 'google', image: false, maxTokens: 8192,
-        options: {
-            // task: 'retrieval.query',
-            // dimensions: 1024,
-            // normalized: true,
-            // late_chunking: true,
-            // embedding_type: 'float',
-        },
+        source: 'google', image: false, maxTokens: 1024, recordsLimit: 200,
     },
 };
 
@@ -122,6 +111,10 @@ const getRerankerClient = (provider) => {
 };
 
 const initEmbedding = async (options = {}) => {
+    if (options?.debug) {
+        (await need('node:util')).inspect.defaultOptions.depth = null;
+        options.logLevel = 'debug';
+    }
     ensureApiKey(options);
     const provider = ensureEmbeddingProvider(options);
     const OpenAI = await need('openai');
@@ -215,20 +208,31 @@ const initReranker = async (options = {}) => {
     return getRerankerClient(provider);
 };
 
-const rerank = async (query, documents, options = {}) => {
+const rerank = async (query, records, options = {}) => {
     assert(query, 'Query is required.', 400);
-    assert(documents?.length, 'Documents are required.', 400);
-    const records = documents.map((doc, idx) => ({
-        id: String(idx), content: doc,
-    }));
+    assert(records?.length, 'Records are required.', 400);
     const { provider, model, client, rankingConfigPath }
         = getRerankerClient(options?.provider);
+    records = records.map((content, id) => Object.isObject(content)
+        ? content : { id: String(id), content }).slice(
+            0, MODEL_CONFIG[model]?.recordsLimit || records.length
+        );
+    const maxTokens = MODEL_CONFIG[model]?.maxTokens || Infinity;
     let result;
+    for (let i in records) {
+        records[i].title = await trimText(records[i]?.title || '', maxTokens);
+        const titleTokens = await countTokens(records[i].title);
+        const availableTokens = maxTokens - titleTokens;
+        records[i].content = availableTokens > 0 ? await trimText(
+            records[i].content, availableTokens
+        ) : '';
+    }
     switch (provider) {
         case GOOGLE:
             const request = {
                 model, query, rankingConfig: rankingConfigPath,
-                records, topN: ~~options?.topN || documents.length,
+                records, topN: ~~options?.topN || records.length,
+                ...options?.requestOptions || {},
             };
             result = (await client.rank(request))?.[0]?.records;
             break;

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "utilitas",
     "description": "Just another common utility for JavaScript.",
-    "version": "2000.3.47",
+    "version": "2000.3.48",
     "private": false,
     "homepage": "https://github.com/Leask/utilitas",
     "main": "index.mjs",