simple-dynamsoft-mcp 7.2.0 → 7.2.1

package/.env.example

@@ -17,9 +17,11 @@
 # Optional: force hydration mode override
 # MCP_DATA_HYDRATION_MODE=eager
 
-# Optional: shared state path for runtime shard loading
-# Set to state/current.json mounted with shard files for all workers.
-# RAG_SHARED_STATE_PATH=/mnt/mcp-cache/state/current.json
+# Optional: prebuilt Gemini index behavior
+# RAG_PREBUILT_INDEX_AUTO_DOWNLOAD=true
+# RAG_PREBUILT_INDEX_URL=
+# RAG_PREBUILT_INDEX_URL_GEMINI=
+# RAG_PREBUILT_INDEX_TIMEOUT_MS=180000
 
 # Optional: prewarm behavior
 # RAG_PREWARM=true

package/README.md

@@ -33,8 +33,7 @@ npx -y simple-dynamsoft-mcp@latest
 
 ## Deployment Guides
 
-- Azure Container Apps runbook (two-lane release/data architecture): `docs/deployment/azure-container-apps.md`
-- Shared Azure data/index sync runbook: `docs/deployment/data-sync-azure.md`
+- Azure Container Apps runbook: `docs/deployment/azure-container-apps.md`
 - Self-hosting (Ubuntu/any server): `docs/deployment/self-hosting.md`
 
 ## MCP Client Configuration

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "simple-dynamsoft-mcp",
-  "version": "7.2.0",
+  "version": "7.2.1",
   "description": "MCP server for Dynamsoft SDKs - Capture Vision, Barcode Reader (Mobile/Python/Web), Dynamic Web TWAIN, and Document Viewer. Provides documentation, code snippets, and API guidance.",
   "license": "MIT",
   "repository": {
@@ -24,7 +24,7 @@
   "scripts": {
     "start": "node src/index.js",
     "test": "npm run test:lite",
-    "test:unit": "node --test test/unit/gemini-retry.test.js test/unit/providers.test.js test/unit/profile-config.test.js test/unit/lexical-provider.test.js test/unit/hydration-mode.test.js test/unit/hydration-policy.test.js test/unit/repo-map.test.js test/unit/download-utils.test.js test/unit/logging.test.js test/unit/rag-logger.test.js test/unit/create-server.test.js test/unit/server-helpers.test.js test/unit/shared-state.test.js test/unit/vector-cache.test.js test/unit/compute-repo-signatures.test.js test/unit/data-sync-azure.test.js",
+    "test:unit": "node --test test/unit/gemini-retry.test.js test/unit/profile-config.test.js test/unit/lexical-provider.test.js test/unit/hydration-mode.test.js test/unit/hydration-policy.test.js test/unit/repo-map.test.js test/unit/download-utils.test.js test/unit/logging.test.js test/unit/create-server.test.js test/unit/server-helpers.test.js",
     "test:lite": "npm run test:stdio && npm run test:http && npm run test:package",
     "test:lexical": "node --test test/integration/stdio.test.js test/integration/http.test.js",
     "test:gemini": "node scripts/run-gemini-tests.mjs",
@@ -65,6 +65,7 @@
   ],
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.25.2",
+    "@xenova/transformers": "^2.17.2",
     "dotenv": "^16.4.5",
     "extract-zip": "^2.0.1",
     "fuse.js": "^7.0.0",

package/src/index.js

@@ -27,8 +27,7 @@ logEvent("profile", "resolved", {
   provider: ragApi.ragConfig.provider,
   provider_source: ragApi.ragConfig.providerSource,
   fallback: ragApi.ragConfig.fallback,
-  fallback_source: ragApi.ragConfig.fallbackSource,
-  shared_state_path: ragApi.ragConfig.sharedStatePath ? "set" : "empty"
+  fallback_source: ragApi.ragConfig.fallbackSource
 });
 
 const createServer = () => createMcpServerInstance({

package/src/rag/config.js

@@ -5,9 +5,18 @@ import { getResolvedDataRoot } from "../data/root.js";
 import { resolveProfileConfig } from "./profile-config.js";
 
 const dataRoot = getResolvedDataRoot();
+
 const pkgUrl = new URL("../../package.json", import.meta.url);
 const pkg = JSON.parse(readFileSync(pkgUrl, "utf8"));
 
+const legacyPrebuiltIndexUrl =
+  `https://github.com/yushulx/simple-dynamsoft-mcp/releases/download/v${pkg.version}/prebuilt-rag-index-${pkg.version}.tar.gz`;
+
+const defaultPrebuiltIndexUrls = {
+  gemini:
+    `https://github.com/yushulx/simple-dynamsoft-mcp/releases/download/v${pkg.version}/prebuilt-rag-index-gemini-${pkg.version}.tar.gz`
+};
+
 function readEnvValue(key, fallback) {
   const value = process.env[key];
   if (value === undefined || value === "") return fallback;
@@ -50,10 +59,8 @@ const ragConfig = {
   fallbackSource: profileConfig.fallbackSource,
   provider: profileConfig.provider,
   fallback: profileConfig.fallback,
-  dataRoot,
   cacheDir: readEnvValue("RAG_CACHE_DIR", join(dataRoot, ".rag-cache")),
   modelCacheDir: readEnvValue("RAG_MODEL_CACHE_DIR", join(dataRoot, ".rag-cache", "models")),
-  sharedStatePath: readEnvValue("RAG_SHARED_STATE_PATH", ""),
   chunkSize: readIntEnv("RAG_CHUNK_SIZE", 1200),
   chunkOverlap: readIntEnv("RAG_CHUNK_OVERLAP", 200),
   maxChunksPerDoc: readIntEnv("RAG_MAX_CHUNKS_PER_DOC", 6),
@@ -63,6 +70,10 @@ const ragConfig = {
   rebuild: readBoolEnv("RAG_REBUILD", false),
   prewarm: readBoolEnv("RAG_PREWARM", defaultPrewarm),
   prewarmBlock: readBoolEnv("RAG_PREWARM_BLOCK", false),
+  prebuiltIndexAutoDownload: readBoolEnv("RAG_PREBUILT_INDEX_AUTO_DOWNLOAD", true),
+  prebuiltIndexUrl: readEnvValue("RAG_PREBUILT_INDEX_URL", ""),
+  prebuiltIndexUrlGemini: readEnvValue("RAG_PREBUILT_INDEX_URL_GEMINI", defaultPrebuiltIndexUrls.gemini),
+  prebuiltIndexTimeoutMs: readIntEnv("RAG_PREBUILT_INDEX_TIMEOUT_MS", 180000),
   geminiApiKey: readEnvValue("GEMINI_API_KEY", ""),
   geminiModel: normalizeGeminiModel(readEnvValue("GEMINI_EMBED_MODEL", "models/gemini-embedding-001")),
   geminiBaseUrl: readEnvValue("GEMINI_API_BASE_URL", "https://generativelanguage.googleapis.com"),
@@ -75,5 +86,7 @@ const ragConfig = {
 
 export {
   pkg,
-  ragConfig
+  ragConfig,
+  legacyPrebuiltIndexUrl,
+  defaultPrebuiltIndexUrls
 };

package/src/rag/gemini-retry.js

@@ -62,7 +62,7 @@ function computeBackoffDelayMs({
 }) {
   const retryAfter = Math.max(0, Number(retryAfterMs) || 0);
   if (retryAfter > 0) {
-    return retryAfter;
+    return Math.min(maxDelayMs, retryAfter);
   }
 
   const exponent = Math.max(0, Number(attempt) - 1);

package/src/rag/index.js

@@ -10,7 +10,7 @@ import {
 } from "../server/resource-index.js";
 import { latencyBucket } from "../observability/logging.js";
 import { createLexicalProvider } from "./lexical-provider.js";
-import { pkg, ragConfig } from "./config.js";
+import { pkg, ragConfig, legacyPrebuiltIndexUrl } from "./config.js";
 import {
   ragLogState,
   logRag,
@@ -27,7 +27,8 @@ import {
   buildEmbeddingItems,
   buildIndexSignature,
   normalizeVector,
-  dotProduct
+  dotProduct,
+  isRateLimitError
 } from "./search-utils.js";
 import { createProviderOrchestrator } from "./providers.js";
 import { createVectorCacheHelpers } from "./vector-cache.js";
@@ -41,6 +42,7 @@ const searchUtils = {
   buildIndexSignature,
   normalizeVector,
   dotProduct,
+  isRateLimitError,
   entryMatchesScope: (entry, filters) => entryMatchesScope(entry, filters, {
     editionMatches,
     platformMatches
@@ -49,6 +51,8 @@ const searchUtils = {
 
 const vectorCache = createVectorCacheHelpers({
   ragConfig,
+  pkgVersion: pkg.version,
+  legacyPrebuiltIndexUrl,
   logRag
 });
 

package/src/rag/logger.js

@@ -9,13 +9,6 @@ const ragLogState = {
   fallbackUse: new Set()
 };
 
-const DEPRECATED_PREBUILT_ENV_KEYS = [
-  "RAG_PREBUILT_INDEX_AUTO_DOWNLOAD",
-  "RAG_PREBUILT_INDEX_URL",
-  "RAG_PREBUILT_INDEX_URL_GEMINI",
-  "RAG_PREBUILT_INDEX_TIMEOUT_MS"
-];
-
 function logRag(eventOrMessage, fields = {}, options = {}) {
   if (fields && typeof fields === "object" && Object.keys(fields).length > 0) {
     logEvent("rag", eventOrMessage, fields, options);
@@ -27,20 +20,12 @@ function logRag(eventOrMessage, fields = {}, options = {}) {
 function logRagConfigOnce(ragConfig) {
   if (ragLogState.config) return;
   ragLogState.config = true;
-  const deprecatedKeys = DEPRECATED_PREBUILT_ENV_KEYS.filter((key) => {
-    const value = process.env[key];
-    return value !== undefined && value !== "";
-  });
-  if (deprecatedKeys.length > 0) {
-    logRag("deprecated_prebuilt_env_vars", {
-      keys: deprecatedKeys.join(","),
-      recommendation: "Use RAG_SHARED_STATE_PATH for shared shard loading."
-    }, { level: "warn" });
-  }
   logRag(
     `config provider=${ragConfig.provider} fallback=${ragConfig.fallback} prewarm=${ragConfig.prewarm} rebuild=${ragConfig.rebuild} ` +
-    `cache_dir=${ragConfig.cacheDir} shared_state_path=${ragConfig.sharedStatePath ? "set" : "empty"} ` +
-    `gemini_retry_max_attempts=${ragConfig.geminiRetryMaxAttempts} ` +
+    `cache_dir=${ragConfig.cacheDir} prebuilt_auto_download=${ragConfig.prebuiltIndexAutoDownload} ` +
+    `prebuilt_url_override=${ragConfig.prebuiltIndexUrl ? "set" : "empty"} ` +
+    `prebuilt_url_gemini=${ragConfig.prebuiltIndexUrlGemini ? "set" : "empty"} ` +
+    `prebuilt_timeout_ms=${ragConfig.prebuiltIndexTimeoutMs} gemini_retry_max_attempts=${ragConfig.geminiRetryMaxAttempts} ` +
     `gemini_retry_base_delay_ms=${ragConfig.geminiRetryBaseDelayMs} gemini_retry_max_delay_ms=${ragConfig.geminiRetryMaxDelayMs} ` +
     `gemini_request_throttle_ms=${ragConfig.geminiRequestThrottleMs}`
   );

package/src/rag/providers.js

@@ -9,18 +9,6 @@ import {
   executeWithGeminiRetry
 } from "./gemini-retry.js";
 
-const GEMINI_EMBEDDING_PAYLOAD_ERROR_CODE = "GEMINI_EMBEDDING_PAYLOAD_INVALID";
-
-function isValidEmbeddingValues(values) {
-  return Array.isArray(values) && values.length > 0;
-}
-
-function createGeminiEmbeddingPayloadError(message) {
-  const error = new Error(message);
-  error.code = GEMINI_EMBEDDING_PAYLOAD_ERROR_CODE;
-  return error;
-}
-
 function resolveProviderChain(ragConfig) {
   let primary = ragConfig.provider;
   if (primary === "auto") {
@@ -43,7 +31,7 @@ async function embedTextsWithProgress(
     onChunk = null,
     providerName = "",
     logRag,
-    isRateLimitError = () => false
+    isRateLimitError
   } = {}
 ) {
   const results = [];
@@ -82,10 +70,6 @@ async function embedTextsWithProgress(
         rateLimitFailures = 0;
         await reportChunk(vectors, "batch", batch.length);
       } catch (error) {
-        if (error?.code === GEMINI_EMBEDDING_PAYLOAD_ERROR_CODE) {
-          throw error;
-        }
-
         if (isRateLimitError(error)) {
           rateLimitFailures += 1;
           const nextBatchSize = Math.max(1, Math.floor(currentBatchSize / 2));
@@ -122,12 +106,12 @@ async function embedTextsWithProgress(
         finalBatchSize: currentBatchSize
       }
     };
-  } else {
-    for (const text of texts) {
-      const vector = await embedder.embed(text);
-      results.push(vector);
-      await reportChunk([vector], "single", 1);
-    }
+  }
+
+  for (const text of texts) {
+    const vector = await embedder.embed(text);
+    results.push(vector);
+    await reportChunk([vector], "single", 1);
   }
 
   return {
@@ -239,8 +223,8 @@ function createProviderOrchestrator({
             }
           );
           const embedding = payload.embedding?.values || payload.embedding || payload.embeddings?.[0]?.values;
-          if (!isValidEmbeddingValues(embedding)) {
-            throw createGeminiEmbeddingPayloadError("Gemini embedding response missing embedding values.");
+          if (!embedding) {
+            throw new Error("Gemini embedding response missing embedding values.");
           }
           return embedding;
         },
@@ -261,16 +245,7 @@ function createProviderOrchestrator({
           if (!Array.isArray(embeddings)) {
             throw new Error("Gemini batch response missing embeddings.");
           }
-
-          return embeddings.map((item, index) => {
-            const values = item?.values || item?.embedding?.values || item?.embedding;
-            if (!isValidEmbeddingValues(values)) {
-              throw createGeminiEmbeddingPayloadError(
-                `Gemini batch embedding response malformed at index=${index}.`
-              );
-            }
-            return values;
-          });
+          return embeddings.map((item) => item.values || item.embedding?.values || item.embedding);
         },
         getMetrics: () => ({ ...metrics }),
         resetMetrics: () => {
@@ -328,32 +303,26 @@ function createProviderOrchestrator({
           }
           logRag(`cache miss provider=${name} file=${cacheFile} reason=${cacheState.reason}`);
 
-          const sharedLoadResult = await vectorCache.maybeLoadSharedVectorIndex({
+          const downloadResult = await vectorCache.maybeDownloadPrebuiltVectorIndex({
             provider: name,
             model,
             cacheKey,
             signature,
             cacheFile
           });
-          if (sharedLoadResult.loaded) {
+          if (downloadResult.downloaded) {
             cacheState = vectorCache.loadVectorIndexCache(cacheFile, expectedCacheState);
             if (cacheState.hit) {
               const cached = cacheState.payload;
               logRag(
-                `cache hit provider=${name} file=${cacheFile} source=shared_state items=${cached.items.length} vectors=${cached.vectors.length}`
+                `cache hit provider=${name} file=${cacheFile} source=prebuilt_download items=${cached.items.length} vectors=${cached.vectors.length}`
               );
               return {
                 items: cached.items,
                 vectors: cached.vectors
               };
             }
-            logRag(`cache miss provider=${name} file=${cacheFile} source=shared_state reason=${cacheState.reason}`);
-          } else if (sharedLoadResult.fatal) {
-            const sharedError = sharedLoadResult.error || new Error(`shared shard load failed (${sharedLoadResult.reason})`);
-            logRag(
-              `shared shard load failed provider=${name} reason=${sharedLoadResult.reason} error=${sharedError.message}`
-            );
-            throw sharedError;
+            logRag(`cache miss provider=${name} file=${cacheFile} source=prebuilt_download reason=${cacheState.reason}`);
           }
         } else {
           logRag(`cache bypass provider=${name} file=${cacheFile} reason=rebuild_true`);
@@ -414,7 +383,7 @@ function createProviderOrchestrator({
                 total: texts.length,
                 providerName: name,
                 logRag,
-                isRateLimitError: (error) => error?.rateLimited || isRateLimitGeminiStatus(Number(error?.status)),
+                isRateLimitError: (error) => utils.isRateLimitError(error, isRateLimitGeminiStatus),
                 onChunk: ({ vectors, completed, total }) => {
                   normalized.push(...vectors.map(utils.normalizeVector));
                   persistCheckpoint(completed >= total);

package/src/rag/search-utils.js

@@ -145,6 +145,12 @@ function dotProduct(a, b) {
   return sum;
 }
 
+function isRateLimitError(error, isRateLimitGeminiStatus) {
+  if (error?.rateLimited) return true;
+  const status = Number(error?.status);
+  return isRateLimitGeminiStatus(status);
+}
+
 export {
   createFuseSearch,
   attachScore,
@@ -155,5 +161,6 @@ export {
   buildEmbeddingItems,
   buildIndexSignature,
   normalizeVector,
-  dotProduct
+  dotProduct,
+  isRateLimitError
 };