simple-dynamsoft-mcp 5.0.0 → 6.3.0

package/scripts/prebuild-rag-index.mjs

@@ -0,0 +1,88 @@
+#!/usr/bin/env node
+import { createHash } from "node:crypto";
+import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
+import { basename, join, resolve } from "node:path";
+
+function toPosixPath(path) {
+  return path.replace(/\\/g, "/");
+}
+
+function fileHash(path) {
+  const data = readFileSync(path);
+  return createHash("sha256").update(data).digest("hex");
+}
+
+function ensureEnvDefaults() {
+  process.env.RAG_PROVIDER = process.env.RAG_PROVIDER || "local";
+  process.env.RAG_FALLBACK = process.env.RAG_FALLBACK || "none";
+  process.env.RAG_PREWARM = process.env.RAG_PREWARM || "true";
+  process.env.RAG_PREWARM_BLOCK = process.env.RAG_PREWARM_BLOCK || "true";
+  process.env.RAG_REBUILD = process.env.RAG_REBUILD || "true";
+}
+
+ensureEnvDefaults();
+
+const pkg = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8"));
+const { prewarmRagIndex, ragConfig } = await import("../src/rag.js");
+
+await prewarmRagIndex();
+
+const cacheDir = resolve(ragConfig.cacheDir);
+if (!existsSync(cacheDir)) {
+  throw new Error(`RAG cache directory not found after prebuild: ${cacheDir}`);
+}
+
+const cacheFiles = readdirSync(cacheDir, { withFileTypes: true })
+  .filter((entry) => entry.isFile())
+  .map((entry) => join(cacheDir, entry.name))
+  .filter((path) => {
+    const name = basename(path).toLowerCase();
+    if (!name.endsWith(".json")) return false;
+    if (!name.startsWith("rag-")) return false;
+    if (name.endsWith(".checkpoint.json")) return false;
+    return true;
+  })
+  .sort();
+
+if (cacheFiles.length === 0) {
+  throw new Error(`No cache JSON files found in ${cacheDir}.`);
+}
+
+let indexSignature = "";
+let indexCacheKey = "";
+const localCacheFile = cacheFiles.find((path) => /rag-local-.*\.json$/i.test(path));
+if (localCacheFile) {
+  try {
+    const parsed = JSON.parse(readFileSync(localCacheFile, "utf8"));
+    indexSignature = String(parsed?.meta?.signature || "");
+    indexCacheKey = String(parsed?.cacheKey || "");
+  } catch {
+    // Keep manifest generation resilient; runtime still validates cache payloads directly.
+  }
+}
+
+const manifest = {
+  packageVersion: pkg.version,
+  generatedAt: new Date().toISOString(),
+  ragProvider: ragConfig.provider,
+  ragModel: ragConfig.provider === "gemini" ? ragConfig.geminiModel : ragConfig.localModel,
+  indexSignature,
+  indexCacheKey,
+  cacheDir: toPosixPath(cacheDir),
+  files: cacheFiles.map((path) => {
+    const stats = statSync(path);
+    return {
+      name: toPosixPath(path.slice(cacheDir.length + 1)),
+      size: stats.size,
+      sha256: fileHash(path)
+    };
+  })
+};
+
+const outputPath = join(cacheDir, "prebuilt-rag-manifest.json");
+mkdirSync(cacheDir, { recursive: true });
+writeFileSync(outputPath, `${JSON.stringify(manifest, null, 2)}\n`);
+
+console.log(`[rag-prebuild] version=${pkg.version}`);
+console.log(`[rag-prebuild] cache_dir=${cacheDir}`);
+console.log(`[rag-prebuild] files=${manifest.files.length}`);

package/scripts/run-gemini-tests.mjs

@@ -0,0 +1,25 @@
+#!/usr/bin/env node
+import { spawn } from "node:child_process";
+
+const child = spawn(
+  process.execPath,
+  ["--test", "test/integration/stdio.test.js", "test/integration/http-gateway.test.js"],
+  {
+    stdio: "inherit",
+    env: {
+      ...process.env,
+      RUN_FUSE_PROVIDER_TESTS: "false",
+      RUN_LOCAL_PROVIDER_TESTS: "false",
+      RUN_GEMINI_PROVIDER_TESTS: "true"
+    }
+  }
+);
+
+child.on("exit", (code, signal) => {
+  if (signal) {
+    process.kill(process.pid, signal);
+    return;
+  }
+  process.exit(code ?? 1);
+});
+

package/scripts/sync-submodules.mjs

@@ -0,0 +1,6 @@
+#!/usr/bin/env node
+import { maybeSyncSubmodulesOnStart } from "../src/submodule-sync.js";
+
+process.env.DATA_SYNC_ON_START = "true";
+console.log("[data-sync] cli wrapper enabled DATA_SYNC_ON_START=true");
+await maybeSyncSubmodulesOnStart();

package/scripts/update-data-lock.mjs

@@ -0,0 +1,124 @@
+#!/usr/bin/env node
+import { readFileSync, writeFileSync, existsSync } from "node:fs";
+import { spawnSync } from "node:child_process";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const projectRoot = join(__dirname, "..");
+const gitmodulesPath = join(projectRoot, ".gitmodules");
+const outputPath = join(projectRoot, "data", "metadata", "data-manifest.json");
+
+function logDataLock(message) {
+  console.log(`[data-lock] ${message}`);
+}
+
+function parseGitmodules(path) {
+  if (!existsSync(path)) return [];
+  const lines = readFileSync(path, "utf8").split(/\r?\n/);
+  const entries = [];
+  let current = null;
+
+  for (const raw of lines) {
+    const line = raw.trim();
+    if (!line) continue;
+
+    const section = line.match(/^\[submodule\s+"([^"]+)"\]$/);
+    if (section) {
+      if (current && current.path && current.url) entries.push(current);
+      current = { name: section[1], path: "", url: "", branch: "main" };
+      continue;
+    }
+
+    if (!current) continue;
+
+    const pathMatch = line.match(/^path\s*=\s*(.+)$/);
+    if (pathMatch) {
+      current.path = pathMatch[1].trim();
+      continue;
+    }
+
+    const urlMatch = line.match(/^url\s*=\s*(.+)$/);
+    if (urlMatch) {
+      current.url = urlMatch[1].trim();
+      continue;
+    }
+
+    const branchMatch = line.match(/^branch\s*=\s*(.+)$/);
+    if (branchMatch) {
+      current.branch = branchMatch[1].trim();
+    }
+  }
+
+  if (current && current.path && current.url) entries.push(current);
+  return entries;
+}
+
+function runGit(args) {
+  const result = spawnSync("git", args, {
+    cwd: projectRoot,
+    encoding: "utf8"
+  });
+  return {
+    ok: result.status === 0,
+    stdout: String(result.stdout || "").trim(),
+    stderr: String(result.stderr || "").trim()
+  };
+}
+
+function parseGitHubUrl(url) {
+  const match = String(url).match(/github\.com[/:]([^/]+)\/([^/.]+)(?:\.git)?$/i);
+  if (!match) return null;
+  return { owner: match[1], repo: match[2] };
+}
+
+function toDataRelativePath(submodulePath) {
+  return submodulePath.replace(/^data[\\/]/, "").replace(/\\/g, "/");
+}
+
+const submodules = parseGitmodules(gitmodulesPath);
+if (submodules.length === 0) {
+  console.error("No submodules found in .gitmodules.");
+  process.exit(1);
+}
+logDataLock(`loaded submodules count=${submodules.length}`);
+
+const repos = [];
+
+for (const entry of submodules) {
+  const slug = parseGitHubUrl(entry.url);
+  if (!slug) {
+    console.error(`Unsupported submodule URL: ${entry.url}`);
+    process.exit(1);
+  }
+
+  const head = runGit(["-C", join(projectRoot, entry.path), "rev-parse", "HEAD"]);
+  if (!head.ok || !head.stdout) {
+    console.error(`Failed to resolve HEAD for ${entry.path}. Run npm run data:bootstrap first.`);
+    if (head.stderr) console.error(head.stderr);
+    process.exit(1);
+  }
+  logDataLock(`resolved path=${entry.path} branch=${entry.branch || "main"} head=${head.stdout}`);
+
+  const relativePath = toDataRelativePath(entry.path);
+  repos.push({
+    name: entry.name,
+    path: relativePath,
+    url: entry.url,
+    branch: entry.branch || "main",
+    owner: slug.owner,
+    repo: slug.repo,
+    commit: head.stdout,
+    archiveUrl: `https://codeload.github.com/${slug.owner}/${slug.repo}/zip/${head.stdout}`
+  });
+}
+
+repos.sort((a, b) => a.path.localeCompare(b.path));
+
+const manifest = {
+  version: 1,
+  repos
+};
+
+writeFileSync(outputPath, `${JSON.stringify(manifest, null, 2)}\n`);
+logDataLock(`updated file=${outputPath} entries=${repos.length}`);

package/scripts/update-sdk-versions.mjs

@@ -0,0 +1,342 @@
+#!/usr/bin/env node
+import { existsSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join, relative } from "node:path";
+
+const projectRoot = process.cwd();
+const metadataPath = join(projectRoot, "data", "metadata", "dynamsoft_sdks.json");
+const checkOnly = process.argv.includes("--check");
+const strictMode = process.argv.includes("--strict") || process.env.VERSION_SYNC_STRICT === "true";
+
+function logVersionSync(message) {
+  console.log(`[version-sync] ${message}`);
+}
+
+function walkFiles(rootDir, fileFilter) {
+  if (!existsSync(rootDir)) return [];
+  const files = [];
+
+  function walk(dir) {
+    for (const entry of readdirSync(dir, { withFileTypes: true })) {
+      if (entry.name.startsWith(".git")) continue;
+      const fullPath = join(dir, entry.name);
+      if (entry.isDirectory()) {
+        walk(fullPath);
+        continue;
+      }
+      if (entry.isFile() && fileFilter(fullPath, entry.name)) {
+        files.push(fullPath);
+      }
+    }
+  }
+
+  walk(rootDir);
+  return files.sort((a, b) => a.localeCompare(b));
+}
+
+function normalizeVersion(version) {
+  const parts = String(version)
+    .trim()
+    .split(".")
+    .map((part) => Number.parseInt(part, 10))
+    .filter((part) => Number.isInteger(part) && part >= 0);
+  if (parts.length < 2) return "";
+  return parts.join(".");
+}
+
+function compareVersion(a, b) {
+  const aParts = String(a).split(".").map((part) => Number.parseInt(part, 10));
+  const bParts = String(b).split(".").map((part) => Number.parseInt(part, 10));
+  const len = Math.max(aParts.length, bParts.length);
+  for (let i = 0; i < len; i += 1) {
+    const av = Number.isInteger(aParts[i]) ? aParts[i] : 0;
+    const bv = Number.isInteger(bParts[i]) ? bParts[i] : 0;
+    if (av > bv) return 1;
+    if (av < bv) return -1;
+  }
+  return 0;
+}
+
+function getHighestVersion(versions) {
+  const normalized = versions
+    .map((item) => normalizeVersion(item))
+    .filter(Boolean);
+  if (normalized.length === 0) return "";
+  normalized.sort((a, b) => compareVersion(a, b)).reverse();
+  return normalized[0];
+}
+
+function extractVersionCandidates(text) {
+  const values = [];
+  const regex = /(?:^|[^0-9])v?(\d{1,2}(?:\.\d+){1,3})(?![\d.])/gi;
+  const source = String(text || "");
+  let match = regex.exec(source);
+  while (match) {
+    const normalized = normalizeVersion(match[1]);
+    if (normalized) values.push(normalized);
+    match = regex.exec(source);
+  }
+  return values;
+}
+
+function detectFromReleaseNoteIndexes(docRoot) {
+  const markdownFiles = walkFiles(docRoot, (fullPath, name) => {
+    if (!name.toLowerCase().endsWith(".md")) return false;
+    const normalized = fullPath.replace(/\\/g, "/").toLowerCase();
+    return normalized.endsWith("/release-notes/index.md") || normalized.endsWith("/releasenotes/index.md");
+  });
+
+  if (markdownFiles.length === 0) {
+    return { version: "", detail: "release-note index files=0" };
+  }
+
+  const versions = [];
+  for (const filePath of markdownFiles) {
+    const content = readFileSync(filePath, "utf8");
+    versions.push(...extractVersionCandidates(content));
+  }
+
+  const version = getHighestVersion(versions);
+  return {
+    version,
+    detail: `release-note index files=${markdownFiles.length} candidates=${versions.length}`
+  };
+}
+
+function detectFromProductVersionYml(docRoot) {
+  const filePath = join(docRoot, "_data", "product_version.yml");
+  if (!existsSync(filePath)) return { version: "", detail: "_data/product_version.yml missing" };
+  const content = readFileSync(filePath, "utf8");
+
+  const latestLine = content
+    .split(/\r?\n/)
+    .find((line) => /latest version/i.test(line));
+  if (!latestLine) return { version: "", detail: "latest version line missing in product_version.yml" };
+
+  const candidates = extractVersionCandidates(latestLine);
+  return {
+    version: getHighestVersion(candidates),
+    detail: `product_version.yml candidates=${candidates.length}`
+  };
+}
+
+function detectFromLatestVersionJs(docRoot, relativeFile) {
+  const filePath = join(docRoot, ...relativeFile.split("/"));
+  if (!existsSync(filePath)) return { version: "", detail: `${relativeFile} missing` };
+  const content = readFileSync(filePath, "utf8");
+  const match = content.match(/versionNoteLatestVersion\s*=\s*["']([0-9.]+)["']/i);
+  if (!match) return { version: "", detail: `${relativeFile} has no versionNoteLatestVersion` };
+  return { version: normalizeVersion(match[1]), detail: `parsed ${relativeFile}` };
+}
+
+function detectFromStrategies(strategies, source, resolvedVersions, metadata) {
+  const docsRoot = source.docsPath ? join(projectRoot, source.docsPath) : "";
+  const attempts = [];
+
+  for (const strategy of strategies) {
+    if (strategy === "release-note-indexes") {
+      if (!docsRoot || !existsSync(docsRoot)) {
+        attempts.push("release-note-indexes: docs root missing");
+        continue;
+      }
+      const result = detectFromReleaseNoteIndexes(docsRoot);
+      attempts.push(`release-note-indexes: ${result.detail}`);
+      if (result.version) {
+        return { version: result.version, strategy: "release-note-indexes", attempts };
+      }
+      continue;
+    }
+
+    if (strategy === "product-version-yml") {
+      if (!docsRoot || !existsSync(docsRoot)) {
+        attempts.push("product-version-yml: docs root missing");
+        continue;
+      }
+      const result = detectFromProductVersionYml(docsRoot);
+      attempts.push(`product-version-yml: ${result.detail}`);
+      if (result.version) {
+        return { version: result.version, strategy: "product-version-yml", attempts };
+      }
+      continue;
+    }
+
+    if (typeof strategy === "object" && strategy.type === "latest-version-js") {
+      if (!docsRoot || !existsSync(docsRoot)) {
+        attempts.push(`latest-version-js(${strategy.file}): docs root missing`);
+        continue;
+      }
+      const result = detectFromLatestVersionJs(docsRoot, strategy.file);
+      attempts.push(`latest-version-js(${strategy.file}): ${result.detail}`);
+      if (result.version) {
+        return { version: result.version, strategy: `latest-version-js(${strategy.file})`, attempts };
+      }
+      continue;
+    }
+
+    if (typeof strategy === "object" && strategy.type === "max-of-sdks") {
+      const candidates = [];
+      for (const sdkId of strategy.sdkIds || []) {
+        const resolved = resolvedVersions[sdkId] || String(metadata?.sdks?.[sdkId]?.version || "");
+        const normalized = normalizeVersion(resolved);
+        if (normalized) candidates.push(normalized);
+      }
+      const detected = getHighestVersion(candidates);
+      attempts.push(`max-of-sdks(${(strategy.sdkIds || []).join(",")}): candidates=${candidates.length}`);
+      if (detected) {
+        return { version: detected, strategy: `max-of-sdks(${(strategy.sdkIds || []).join(",")})`, attempts };
+      }
+      continue;
+    }
+  }
+
+  return { version: "", strategy: "", attempts };
+}
+
+const sdkVersionSources = [
+  {
+    sdkId: "dbr-web",
+    docsPath: "data/documentation/barcode-reader-docs-js",
+    strategies: ["release-note-indexes"]
+  },
+  {
+    sdkId: "dbr-mobile",
+    docsPath: "data/documentation/barcode-reader-docs-mobile",
+    strategies: ["release-note-indexes"]
+  },
+  {
+    sdkId: "dbr-server",
+    docsPath: "data/documentation/barcode-reader-docs-server",
+    strategies: ["release-note-indexes"]
+  },
+  {
+    sdkId: "dwt",
+    docsPath: "data/documentation/web-twain-docs",
+    strategies: [{ type: "latest-version-js", file: "assets/js/setLatestVersion.js" }]
+  },
+  {
+    sdkId: "ddv",
+    docsPath: "data/documentation/document-viewer-docs",
+    strategies: ["product-version-yml", "release-note-indexes"]
+  },
+  {
+    sdkId: "dcv-web",
+    docsPath: "data/documentation/capture-vision-docs-js",
+    strategies: ["release-note-indexes"]
+  },
+  {
+    sdkId: "dcv-mobile",
+    docsPath: "data/documentation/capture-vision-docs-mobile",
+    strategies: ["release-note-indexes"]
+  },
+  {
+    sdkId: "dcv-server",
+    docsPath: "data/documentation/capture-vision-docs-server",
+    strategies: ["release-note-indexes"]
+  },
+  {
+    sdkId: "dcv-core",
+    docsPath: "data/documentation/capture-vision-docs",
+    strategies: ["product-version-yml", { type: "max-of-sdks", sdkIds: ["dcv-server", "dcv-mobile", "dcv-web"] }]
+  }
+];
+
+if (!existsSync(metadataPath)) {
+  console.error(`[version-sync] metadata file not found: ${metadataPath}`);
+  process.exit(1);
+}
+
+logVersionSync(
+  `start mode=${checkOnly ? "check" : "update"} strict=${strictMode ? "true" : "false"} metadata=${relative(projectRoot, metadataPath)} sources=${sdkVersionSources.length}`
+);
+
+const metadata = JSON.parse(readFileSync(metadataPath, "utf8"));
+const updates = [];
+const unchanged = [];
+const skipped = [];
+const resolvedVersions = {};
+
+for (const source of sdkVersionSources) {
+  const sdkEntry = metadata?.sdks?.[source.sdkId];
+  if (!sdkEntry) {
+    skipped.push(`${source.sdkId} (missing metadata entry)`);
+    continue;
+  }
+
+  if (source.docsPath) {
+    const docsRoot = join(projectRoot, source.docsPath);
+    if (!existsSync(docsRoot)) {
+      skipped.push(`${source.sdkId} (${source.docsPath} not found)`);
+      continue;
+    }
+  }
+
+  const detection = detectFromStrategies(source.strategies, source, resolvedVersions, metadata);
+  if (!detection.version) {
+    skipped.push(
+      `${source.sdkId} (no version found; attempts=${detection.attempts.join(" | ") || "none"})`
+    );
+    continue;
+  }
+
+  resolvedVersions[source.sdkId] = detection.version;
+  const current = String(sdkEntry.version || "");
+  if (current !== detection.version) {
+    updates.push({
+      sdkId: source.sdkId,
+      from: current,
+      to: detection.version,
+      docsPath: source.docsPath || "n/a",
+      strategy: detection.strategy
+    });
+    sdkEntry.version = detection.version;
+  } else {
+    unchanged.push({
+      sdkId: source.sdkId,
+      version: current,
+      strategy: detection.strategy
+    });
+  }
+}
+
+if (updates.length === 0) {
+  logVersionSync("no version updates detected");
+} else {
+  for (const update of updates) {
+    logVersionSync(
+      `update ${update.sdkId}: ${update.from} -> ${update.to} (source=${update.docsPath}, strategy=${update.strategy})`
+    );
+  }
+}
+
+if (unchanged.length > 0) {
+  for (const item of unchanged) {
+    logVersionSync(`keep ${item.sdkId}: ${item.version} (strategy=${item.strategy})`);
+  }
+}
+
+if (skipped.length > 0) {
+  for (const item of skipped) {
+    logVersionSync(`skip ${item}`);
+  }
+}
+
+logVersionSync(
+  `summary updates=${updates.length} unchanged=${unchanged.length} skipped=${skipped.length}`
+);
+
+if (strictMode && skipped.length > 0) {
+  console.error("[version-sync] strict mode failed: one or more SDK sources could not be resolved.");
+  process.exit(1);
+}
+
+if (checkOnly) {
+  if (updates.length > 0) {
+    console.error("[version-sync] version metadata is stale. Run: npm run data:versions");
+    process.exit(1);
+  }
+  process.exit(0);
+}
+
+if (updates.length > 0) {
+  writeFileSync(metadataPath, `${JSON.stringify(metadata, null, 2)}\n`);
+  logVersionSync(`updated ${relative(projectRoot, metadataPath)}`);
+}