simple-dynamsoft-mcp 7.2.0 → 7.2.2

package/src/rag/vector-cache.js

@@ -1,11 +1,8 @@
-import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
-import { dirname, isAbsolute, join, resolve } from "node:path";
-import {
-  computeRepoSignature,
-  loadSharedState,
-  normalizeRepoKey,
-  normalizeRepoPath
-} from "../data/shared-state.js";
+import { readFileSync, existsSync, writeFileSync, mkdirSync, readdirSync, copyFileSync, rmSync, statSync } from "node:fs";
+import { join, basename, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { tmpdir } from "node:os";
+import * as tar from "tar";
 
 function ensureDirectory(path) {
   if (!existsSync(path)) {
@@ -105,197 +102,206 @@ function clearVectorIndexCheckpoint(checkpointFile) {
   }
 }
 
-function readManifestRepos(dataRoot) {
-  if (!dataRoot) return [];
-  const manifestPath = join(dataRoot, "metadata", "data-manifest.json");
-  if (!existsSync(manifestPath)) return [];
+function listFilesRecursive(rootDir) {
+  const files = [];
+  const stack = [rootDir];
+  while (stack.length > 0) {
+    const current = stack.pop();
+    const entries = readdirSync(current, { withFileTypes: true });
+    for (const entry of entries) {
+      const fullPath = join(current, entry.name);
+      if (entry.isDirectory()) {
+        stack.push(fullPath);
+      } else if (entry.isFile()) {
+        files.push(fullPath);
+      }
+    }
+  }
+  return files;
+}
 
+function readSignaturePackageVersion(signatureRaw) {
+  if (!signatureRaw) return "";
   try {
-    const parsed = JSON.parse(readFileSync(manifestPath, "utf8"));
-    if (!Array.isArray(parsed?.repos)) return [];
-    return parsed.repos
-      .map((repo) => ({
-        path: normalizeRepoPath(repo?.path),
-        commit: String(repo?.commit || "").trim()
-      }))
-      .filter((repo) => repo.path && repo.commit);
+    const parsed = JSON.parse(signatureRaw);
+    return String(parsed?.packageVersion || "");
   } catch {
-    return [];
+    return "";
   }
 }
 
-function resolveSharedShardFile(sharedStatePath, shardPath) {
-  const rawShardPath = String(shardPath || "").trim();
-  if (!rawShardPath) {
-    throw new Error("shared shard path is empty");
-  }
-  if (isAbsolute(rawShardPath)) {
-    return rawShardPath;
+function listDownloadedCacheCandidatesByProvider(extractRoot, expectedCacheFileName, cacheKey, provider) {
+  const allFiles = listFilesRecursive(extractRoot).filter((path) => path.toLowerCase().endsWith(".json")).sort();
+  const expectedPath = allFiles.find((path) => basename(path) === expectedCacheFileName);
+
+  const cachePrefix = cacheKey.slice(0, 12);
+  const prefixPath = allFiles.find((path) => {
+    const name = basename(path);
+    return name.startsWith(`rag-${provider}-`) && name.endsWith(`-${cachePrefix}.json`);
+  });
+
+  const providerFiles = allFiles.filter((path) => basename(path).startsWith(`rag-${provider}-`));
+  const unique = [];
+  for (const path of [expectedPath, prefixPath, ...providerFiles]) {
+    if (!path) continue;
+    if (!unique.includes(path)) unique.push(path);
   }
+  return unique;
+}
 
-  const rawSegments = rawShardPath.replace(/\\/g, "/").split("/").filter(Boolean);
-  if (rawSegments.includes("..")) {
-    throw new Error("shared shard path must not include parent traversal segments");
-  }
+function resolvePrebuiltIndexUrlCandidates(provider, ragConfig, legacyPrebuiltIndexUrl) {
+  const override = String(ragConfig.prebuiltIndexUrl || "").trim();
+  if (override) return [override];
 
-  const normalizedShardPath = normalizeRepoPath(rawShardPath);
-  if (!normalizedShardPath) {
-    throw new Error("shared shard path is empty");
+  const candidates = [];
+  if (provider === "gemini") {
+    candidates.push(String(ragConfig.prebuiltIndexUrlGemini || "").trim());
   }
+  candidates.push(legacyPrebuiltIndexUrl);
 
-  const stateDir = dirname(sharedStatePath);
-  const stateRoot = resolve(stateDir, "..");
-  const workspaceRoot = dirname(stateRoot);
-  const candidates = [
-    resolve(stateRoot, normalizedShardPath),
-    resolve(stateDir, normalizedShardPath),
-    resolve(workspaceRoot, normalizedShardPath),
-    resolve(process.cwd(), normalizedShardPath)
-  ];
-
+  const deduped = [];
   for (const candidate of candidates) {
-    if (existsSync(candidate)) {
-      return candidate;
-    }
+    if (!candidate) continue;
+    if (!deduped.includes(candidate)) deduped.push(candidate);
   }
-
-  return candidates[0];
+  return deduped;
 }
 
-function parseSharedShardPayload(shardPath, raw) {
-  const parsed = JSON.parse(raw);
-  const candidate = parsed?.items && parsed?.vectors ? parsed : parsed?.payload;
+async function downloadPrebuiltArchive(url, outputPath, timeoutMs) {
+  const source = String(url || "").trim();
+  if (!source) {
+    throw new Error("prebuilt URL is empty");
+  }
 
-  if (!candidate || !Array.isArray(candidate.items) || !Array.isArray(candidate.vectors)) {
-    throw new Error(`shared shard ${shardPath} payload is missing items/vectors arrays`);
+  if (source.startsWith("file://")) {
+    copyFileSync(fileURLToPath(source), outputPath);
+    return { sourceType: "file", size: statSync(outputPath).size };
   }
-  if (candidate.items.length !== candidate.vectors.length) {
-    throw new Error(
-      `shared shard ${shardPath} has mismatched items/vectors lengths (${candidate.items.length}/${candidate.vectors.length})`
-    );
+
+  if (!/^https?:\/\//i.test(source)) {
+    copyFileSync(resolve(source), outputPath);
+    return { sourceType: "file", size: statSync(outputPath).size };
   }
 
-  return {
-    items: candidate.items,
-    vectors: candidate.vectors
-  };
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), Math.max(1000, timeoutMs));
+  try {
+    const response = await fetch(source, { signal: controller.signal });
+    if (!response.ok) {
+      throw new Error(`HTTP ${response.status}`);
+    }
+    const arrayBuffer = await response.arrayBuffer();
+    writeFileSync(outputPath, Buffer.from(arrayBuffer));
+    return { sourceType: "http", size: arrayBuffer.byteLength };
+  } finally {
+    clearTimeout(timer);
+  }
 }
 
-function buildSharedIndexConfig(ragConfig) {
-  return {
-    chunkSize: ragConfig.chunkSize,
-    chunkOverlap: ragConfig.chunkOverlap,
-    maxChunksPerDoc: ragConfig.maxChunksPerDoc,
-    maxTextChars: ragConfig.maxTextChars
-  };
-}
+function createVectorCacheHelpers({ ragConfig, pkgVersion, legacyPrebuiltIndexUrl, logRag }) {
+  const prebuiltDownloadAttempts = new Map();
 
-function createVectorCacheHelpers({ ragConfig, logRag }) {
-  async function maybeLoadSharedVectorIndex({ provider, model, cacheKey, signature, cacheFile }) {
+  async function maybeDownloadPrebuiltVectorIndex({ provider, model, cacheKey, signature, cacheFile }) {
     if (provider !== "gemini") {
-      return { loaded: false, reason: "provider_not_supported" };
-    }
-
-    const sharedStatePath = String(ragConfig.sharedStatePath || "").trim();
-    if (!sharedStatePath) {
-      return { loaded: false, reason: "shared_state_not_configured" };
+      return { downloaded: false, reason: "provider_not_supported" };
     }
-
-    if (!existsSync(sharedStatePath)) {
-      return { loaded: false, reason: "shared_state_unreadable" };
-    }
-
-    let sharedState;
-    try {
-      sharedState = loadSharedState(readFileSync(sharedStatePath, "utf8"));
-    } catch (error) {
-      const normalizedError = error instanceof Error ? error : new Error(String(error));
-      return {
-        loaded: false,
-        fatal: true,
-        reason: "shared_state_invalid",
-        error: normalizedError
-      };
+    if (!ragConfig.prebuiltIndexAutoDownload) {
+      return { downloaded: false, reason: "auto_download_disabled" };
     }
 
-    const manifestRepos = readManifestRepos(ragConfig.dataRoot);
-    if (manifestRepos.length === 0) {
-      return { loaded: false, reason: "manifest_missing_or_empty" };
+    const sourceUrls = resolvePrebuiltIndexUrlCandidates(provider, ragConfig, legacyPrebuiltIndexUrl);
+    if (sourceUrls.length === 0) {
+      return { downloaded: false, reason: "url_not_set" };
     }
 
-    const repoIndexConfig = buildSharedIndexConfig(ragConfig);
-    const matchedShards = [];
-    for (const repo of manifestRepos) {
-      const repoKey = normalizeRepoKey(repo.path);
-      const sharedRepo = sharedState.repos[repoKey];
-      if (!sharedRepo) continue;
-
-      const expectedSignature = computeRepoSignature({
-        repo,
-        embeddingModel: model,
-        indexConfig: repoIndexConfig,
-        indexVersion: sharedState.indexVersion
-      });
-
-      if (sharedRepo.signature !== expectedSignature) continue;
-      matchedShards.push({
-        repoPath: repo.path,
-        shardPath: sharedRepo.shardPath,
-        signature: expectedSignature
-      });
+    const attemptKey = `${provider}:${cacheKey}:${sourceUrls.join("|")}`;
+    if (prebuiltDownloadAttempts.has(attemptKey)) {
+      return prebuiltDownloadAttempts.get(attemptKey);
     }
 
-    if (matchedShards.length === 0) {
-      return { loaded: false, reason: "no_matching_shared_shards" };
-    }
-
-    const combinedItems = [];
-    const combinedVectors = [];
-
-    try {
-      for (const shard of matchedShards) {
-        const shardFile = resolveSharedShardFile(sharedStatePath, shard.shardPath);
-        if (!existsSync(shardFile)) {
-          throw new Error(`shared shard missing repo=${shard.repoPath} path=${shard.shardPath}`);
+    const expectedCacheFileName = makeCacheFileName(provider, model, cacheKey);
+    const attempt = (async () => {
+      let lastReason = "not_attempted";
+      for (const sourceUrl of sourceUrls) {
+        const tempRoot = join(
+          tmpdir(),
+          `simple-dynamsoft-mcp-rag-prebuilt-${Date.now()}-${Math.random().toString(16).slice(2)}`
+        );
+        const archivePath = join(tempRoot, "prebuilt-rag-index.tar.gz");
+        const extractRoot = join(tempRoot, "extract");
+
+        ensureDirectory(extractRoot);
+        try {
+          logRag(
+            `prebuilt index download start provider=${provider} url=${sourceUrl} timeout_ms=${ragConfig.prebuiltIndexTimeoutMs}`
+          );
+          const downloaded = await downloadPrebuiltArchive(sourceUrl, archivePath, ragConfig.prebuiltIndexTimeoutMs);
+          logRag(
+            `prebuilt index downloaded provider=${provider} source=${downloaded.sourceType} size=${downloaded.size}B url=${sourceUrl}`
+          );
+
+          await tar.x({
+            file: archivePath,
+            cwd: extractRoot,
+            strict: true
+          });
+
+          const candidateFiles = listDownloadedCacheCandidatesByProvider(
+            extractRoot,
+            expectedCacheFileName,
+            cacheKey,
+            provider
+          );
+          if (candidateFiles.length === 0) {
+            throw new Error(`cache_file_not_found expected=${expectedCacheFileName}`);
+          }
+
+          for (const sourceCacheFile of candidateFiles) {
+            const candidateCache = loadVectorIndexCache(sourceCacheFile, {
+              provider,
+              model
+            });
+            if (!candidateCache.hit) {
+              continue;
+            }
+
+            const cachePackageVersion = readSignaturePackageVersion(candidateCache.payload?.meta?.signature);
+            if (!cachePackageVersion || cachePackageVersion !== pkgVersion) {
+              continue;
+            }
+
+            const migratedPayload = {
+              ...candidateCache.payload,
+              cacheKey,
+              meta: {
+                ...(candidateCache.payload.meta || {}),
+                provider,
+                model,
+                signature
+              }
+            };
+            saveVectorIndexCache(ragConfig.cacheDir, cacheFile, migratedPayload);
+            logRag(
+              `prebuilt index installed provider=${provider} cache_file=${cacheFile} source=${basename(sourceCacheFile)} mode=version_only_compat version=${cachePackageVersion}`
+            );
+            return { downloaded: true, reason: "installed_version_only_compat" };
+          }
+
+          throw new Error(
+            `no_compatible_cache expected=${expectedCacheFileName} found=${candidateFiles.map((path) => basename(path)).join(",")}`
+          );
+        } catch (error) {
+          lastReason = `${sourceUrl} => ${error.message}`;
+          logRag(`prebuilt index unavailable provider=${provider} url=${sourceUrl} reason=${error.message}`);
+        } finally {
+          rmSync(tempRoot, { recursive: true, force: true });
         }
-        const parsed = parseSharedShardPayload(shard.shardPath, readFileSync(shardFile, "utf8"));
-        combinedItems.push(...parsed.items);
-        combinedVectors.push(...parsed.vectors);
       }
-    } catch (error) {
-      const normalizedError = error instanceof Error ? error : new Error(String(error));
-      return {
-        loaded: false,
-        fatal: true,
-        reason: "shared_shard_error",
-        error: normalizedError
-      };
-    }
+      return { downloaded: false, reason: lastReason };
+    })();
 
-    const payload = {
-      cacheKey,
-      meta: {
-        provider,
-        model,
-        signature,
-        source: "shared_state",
-        sharedStatePath,
-        sharedStateIndexVersion: sharedState.indexVersion,
-        sharedShardCount: matchedShards.length
-      },
-      items: combinedItems,
-      vectors: combinedVectors
-    };
-    saveVectorIndexCache(ragConfig.cacheDir, cacheFile, payload);
-    logRag(
-      `shared shard index loaded provider=${provider} shards=${matchedShards.length} items=${combinedItems.length} vectors=${combinedVectors.length}`
-    );
-    return {
-      loaded: true,
-      reason: "loaded_shared_shards",
-      shardCount: matchedShards.length,
-      itemCount: combinedItems.length
-    };
+    prebuiltDownloadAttempts.set(attemptKey, attempt);
+    return attempt;
   }
 
   return {
@@ -306,11 +312,7 @@ function createVectorCacheHelpers({ ragConfig, logRag }) {
     loadVectorIndexCheckpoint,
     saveVectorIndexCheckpoint: (checkpointFile, payload) => saveVectorIndexCheckpoint(ragConfig.cacheDir, checkpointFile, payload),
     clearVectorIndexCheckpoint,
-    maybeLoadSharedVectorIndex,
-    maybeDownloadPrebuiltVectorIndex: async () => ({
-      downloaded: false,
-      reason: "runtime_prebuilt_download_disabled"
-    })
+    maybeDownloadPrebuiltVectorIndex
   };
 }
 

package/scripts/compute-repo-signatures.mjs

@@ -1,210 +0,0 @@
-#!/usr/bin/env node
-import { existsSync, readFileSync, writeFileSync } from "node:fs";
-import { dirname, join } from "node:path";
-import { fileURLToPath, pathToFileURL } from "node:url";
-
-import {
-  SHARED_STATE_SCHEMA_VERSION,
-  computeRepoSignature,
-  createSharedState,
-  normalizeRepoKey,
-  normalizeRepoPath
-} from "../src/data/shared-state.js";
-
-const __dirname = dirname(fileURLToPath(import.meta.url));
-const projectRoot = join(__dirname, "..");
-const DEFAULT_EMBEDDING_MODEL = "models/gemini-embedding-001";
-const DEFAULT_INDEX_VERSION = "azure-shared-v1";
-const DEFAULT_INDEX_CONFIG = {
-  chunkSize: 1200,
-  chunkOverlap: 200,
-  maxChunksPerDoc: 6,
-  maxTextChars: 4000
-};
-
-function parseIntegerOption(flag, rawValue, { min = 0 } = {}) {
-  const valueText = String(rawValue ?? "").trim();
-  if (!/^-?\d+$/.test(valueText)) {
-    throw new Error(`Invalid ${flag}: expected an integer, received '${rawValue}'`);
-  }
-
-  const value = Number.parseInt(valueText, 10);
-  if (!Number.isSafeInteger(value)) {
-    throw new Error(`Invalid ${flag}: expected a safe integer, received '${rawValue}'`);
-  }
-  if (value < min) {
-    throw new Error(`Invalid ${flag}: expected integer >= ${min}, received '${rawValue}'`);
-  }
-
-  return value;
-}
-
-function parseArgs(argv, env = process.env) {
-  const defaultIndexConfig = {
-    chunkSize: env.DATA_SYNC_AZURE_CHUNK_SIZE
-      ? parseIntegerOption("--chunk-size", env.DATA_SYNC_AZURE_CHUNK_SIZE, { min: 0 })
-      : DEFAULT_INDEX_CONFIG.chunkSize,
-    chunkOverlap: env.DATA_SYNC_AZURE_CHUNK_OVERLAP
-      ? parseIntegerOption("--chunk-overlap", env.DATA_SYNC_AZURE_CHUNK_OVERLAP, { min: 0 })
-      : DEFAULT_INDEX_CONFIG.chunkOverlap,
-    maxChunksPerDoc: env.DATA_SYNC_AZURE_MAX_CHUNKS_PER_DOC
-      ? parseIntegerOption("--max-chunks-per-doc", env.DATA_SYNC_AZURE_MAX_CHUNKS_PER_DOC, { min: 1 })
-      : DEFAULT_INDEX_CONFIG.maxChunksPerDoc,
-    maxTextChars: env.DATA_SYNC_AZURE_MAX_TEXT_CHARS
-      ? parseIntegerOption("--max-text-chars", env.DATA_SYNC_AZURE_MAX_TEXT_CHARS, { min: 0 })
-      : DEFAULT_INDEX_CONFIG.maxTextChars
-  };
-
-  const args = {
-    manifest: "data/metadata/data-manifest.json",
-    output: "",
-    embeddingModel: env.DATA_SYNC_AZURE_EMBEDDING_MODEL || DEFAULT_EMBEDDING_MODEL,
-    indexVersion: env.DATA_SYNC_AZURE_INDEX_VERSION || DEFAULT_INDEX_VERSION,
-    generatedAt: new Date().toISOString(),
-    schemaVersion: SHARED_STATE_SCHEMA_VERSION,
-    indexConfig: defaultIndexConfig
-  };
-
-  for (let i = 0; i < argv.length; i++) {
-    const arg = argv[i];
-    const value = argv[i + 1];
-
-    if (arg === "--manifest" && value) {
-      args.manifest = value;
-      i++;
-      continue;
-    }
-    if (arg === "--output" && value) {
-      args.output = value;
-      i++;
-      continue;
-    }
-    if (arg === "--embedding-model" && value) {
-      args.embeddingModel = value;
-      i++;
-      continue;
-    }
-    if (arg === "--index-version" && value) {
-      args.indexVersion = String(value).trim();
-      i++;
-      continue;
-    }
-    if (arg === "--generated-at" && value) {
-      args.generatedAt = value;
-      i++;
-      continue;
-    }
-    if (arg === "--schema-version" && value) {
-      args.schemaVersion = parseIntegerOption("--schema-version", value, { min: 1 });
-      i++;
-      continue;
-    }
-    if (arg === "--chunk-size" && value) {
-      args.indexConfig.chunkSize = parseIntegerOption("--chunk-size", value, { min: 0 });
-      i++;
-      continue;
-    }
-    if (arg === "--chunk-overlap" && value) {
-      args.indexConfig.chunkOverlap = parseIntegerOption("--chunk-overlap", value, { min: 0 });
-      i++;
-      continue;
-    }
-    if (arg === "--max-chunks-per-doc" && value) {
-      args.indexConfig.maxChunksPerDoc = parseIntegerOption("--max-chunks-per-doc", value, { min: 1 });
-      i++;
-      continue;
-    }
-    if (arg === "--max-text-chars" && value) {
-      args.indexConfig.maxTextChars = parseIntegerOption("--max-text-chars", value, { min: 0 });
-      i++;
-    }
-  }
-
-  return args;
-}
-
-function toAbsolutePath(relativeOrAbsolutePath) {
-  if (!relativeOrAbsolutePath) return "";
-  if (relativeOrAbsolutePath.startsWith("/")) return relativeOrAbsolutePath;
-  return join(projectRoot, relativeOrAbsolutePath);
-}
-
-function loadManifest(pathToManifest) {
-  if (!existsSync(pathToManifest)) {
-    throw new Error(`Manifest not found: ${pathToManifest}`);
-  }
-  const manifest = JSON.parse(readFileSync(pathToManifest, "utf8"));
-  if (!Array.isArray(manifest?.repos)) {
-    throw new Error("Manifest must include a repos array");
-  }
-  return manifest;
-}
-
-function buildShardPath(repoPath, signature) {
-  return `rag/cache/gemini-${signature}.json`;
-}
-
-function buildReposState(manifestRepos, options) {
-  const repos = {};
-  const sourceByKey = new Map();
-
-  for (const repo of manifestRepos) {
-    const path = normalizeRepoPath(repo.path);
-    const key = normalizeRepoKey(path);
-    if (!key) continue;
-
-    const existingPath = sourceByKey.get(key);
-    if (existingPath && existingPath !== path) {
-      throw new Error(
-        `Repo key collision for '${key}': '${existingPath}' and '${path}' normalize to the same key`
-      );
-    }
-    sourceByKey.set(key, path);
-
-    const signature = computeRepoSignature({
-      repo,
-      embeddingModel: options.embeddingModel,
-      indexConfig: options.indexConfig,
-      indexVersion: options.indexVersion,
-      schemaVersion: options.schemaVersion
-    });
-
-    repos[key] = {
-      path,
-      commit: String(repo.commit || "").trim(),
-      signature,
-      shardPath: buildShardPath(path, signature)
-    };
-  }
-
-  return repos;
-}
-
-function main() {
-  const options = parseArgs(process.argv.slice(2));
-  const manifestPath = toAbsolutePath(options.manifest);
-  const outputPath = toAbsolutePath(options.output);
-  const manifest = loadManifest(manifestPath);
-
-  const state = createSharedState({
-    schemaVersion: options.schemaVersion,
-    generatedAt: options.generatedAt,
-    indexVersion: options.indexVersion,
-    repos: buildReposState(manifest.repos, options)
-  });
-
-  const output = `${JSON.stringify(state, null, 2)}\n`;
-  if (outputPath) {
-    writeFileSync(outputPath, output);
-    console.log(`Wrote shared state to ${outputPath}`);
-    return;
-  }
-
-  process.stdout.write(output);
-}
-
-if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) {
-  main();
-}
-
-export { buildReposState, parseArgs, parseIntegerOption };