sift-light 1.0.3-5 → 1.0.3-6

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 1.0.3-6 — 2026-10-09
+
+- Add bounded, read-only configured data-source adapters with explicit HTTPS/loopback validation, credential references and revision-aware source reads.
+- Improve hybrid relevance ranking with request-scoped BM25 and semantic rank fusion, identifier-declaration candidates and distinct passage handling.
+- Harden MCP/request-contract integration and add a packaged MCP smoke test so the published artifact is exercised through the real stdio transport.
+
 ## 1.0.3-5 — 2026-10-07
 
 - Update the MCP SDK to 1.31.0 and Transformers.js to 4.3.1 to remove the dependency vulnerabilities reported in [#111](https://github.com/lightsifter/sift-light/issues/111), including the old ONNX proxy/logging dependency chain. Refresh compatible SDK transitive dependencies in the lockfile.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sift-light",
-  "version": "1.0.3-5",
+  "version": "1.0.3-6",
   "description": "Context-efficient local search for files, documents, notes and logs across Pi, OMP and MCP clients",
   "keywords": [
     "ai-agent",
@@ -38,6 +38,7 @@
     "src/syntax-worker.mjs",
     "src/mcp-server.mjs",
     "src/syntax-worker.toml",
+    "src/package-smoke.mjs",
     "README.md",
     "README.zh-CN.md",
     "CONTRIBUTING.md",
@@ -62,8 +63,9 @@
     "format:check": "oxfmt --check .",
     "lint": "oxlint --type-aware --deny-warnings --report-unused-disable-directives src scripts",
     "typecheck": "tsc --noEmit",
-    "test": "bun test ./doc/testing/test",
-    "test:node": "bun run doc/testing/scripts/node-smoke.ts",
+    "test:repo": "bun test ./doc/testing/test",
+    "test": "node src/package-smoke.mjs",
+    "test:node:repo": "bun run doc/testing/scripts/node-smoke.ts",
     "benchmark": "bun run doc/testing/scripts/benchmark.ts",
     "check": "bun run format:check && bun run lint && bun run typecheck && bun run build",
     "pack:check": "bun pm pack --dry-run && bun run scripts/check-npm-publish.ts",
@@ -71,6 +73,7 @@
     "build:concept-worker": "bun run scripts/build-concept-worker.ts",
     "check:concept-worker": "bun run doc/testing/scripts/check-concept-worker.ts",
     "test:concept": "bun run doc/testing/scripts/test-concept.ts",
+    "evaluate:relevance": "bun run doc/testing/scripts/relevance-eval.ts",
     "evaluate:investigations": "bun run doc/testing/scripts/investigation-eval.ts",
     "build:search-plugin": "bun run scripts/build-search-plugin.ts",
     "check:search-plugin": "bun run doc/testing/scripts/check-search-plugin.ts",
@@ -79,7 +82,7 @@
     "format:local:check": "bun run doc/testing/scripts/format-local.ts --check",
     "lint:local": "oxlint --no-ignore --type-aware --tsconfig doc/testing/tsconfig.json --deny-warnings --report-unused-disable-directives src scripts doc/testing/test doc/testing/scripts",
     "typecheck:local": "tsc --noEmit --project doc/testing/tsconfig.json",
-    "check:local": "bun run format:check && bun run format:local:check && bun run check:search-plugin && bun run check:concept-worker && bun run check:worker && bun run check:mcp && bun run lint:local && bun run typecheck:local && bun run test && bun run test:node && bun run benchmark"
+    "check:local": "bun run format:check && bun run format:local:check && bun run check:search-plugin && bun run check:concept-worker && bun run check:worker && bun run check:mcp && bun run lint:local && bun run typecheck:local && bun run test:repo && bun run test:node:repo && bun run benchmark"
   },
   "dependencies": {
     "@ast-grep/lang-go": "0.0.6",

package/plugins/sift-light/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "sift-light",
-  "version": "1.0.3-5",
+  "version": "1.0.3-6",
   "description": "Require sift-light for conventional local searches while keeping development tools available.",
   "author": {
     "name": "baoer"

package/plugins/sift-light/.codex-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "sift-light",
-  "version": "1.0.3-5",
+  "version": "1.0.3-6",
   "description": "Require sift-light for conventional local searches while keeping development tools available.",
   "author": {
     "name": "baoer"

package/plugins/sift-light/.mcp.json

@@ -5,7 +5,7 @@
       "args": [
         "--yes",
         "--package",
-        "sift-light-runtime@npm:sift-light@1.0.3-5",
+        "sift-light-runtime@npm:sift-light@1.0.3-6",
         "sift-light-mcp",
         "--stdio"
       ],

package/plugins/sift-light/concept-worker.mjs

@@ -462,8 +462,12 @@ async function installConceptModel() {
 
 // src/concept-worker.ts
 var CACHE_IO_CONCURRENCY = 64;
+var activeRequestId;
+var extractor;
+var modelVerified = false;
+var modelLoads = 0;
 function emitProgress(progress) {
-  process.stdout.write(`${JSON.stringify({ type: "progress", ...progress })}
+  process.stdout.write(`${JSON.stringify({ type: "progress", id: activeRequestId, ...progress })}
 `);
 }
 async function readCachedEmbeddings(root, requested, onProgress) {
@@ -475,7 +479,7 @@ async function readCachedEmbeddings(root, requested, onProgress) {
   }
   return cached;
 }
-async function requestFromStdin() {
+async function readSingleRequest() {
   const chunks = [];
   let bytes = 0;
   for await (const chunk of process.stdin) {
@@ -485,7 +489,9 @@ async function requestFromStdin() {
     if (bytes > MAX_CONCEPT_WORKER_INPUT_BYTES)
       throw new Error("Concept worker input exceeds its 64 MiB source protocol budget");
   }
-  const request = JSON.parse(Buffer.concat(chunks).toString("utf8"));
+  return JSON.parse(Buffer.concat(chunks).toString("utf8"));
+}
+function decodeRequest(request) {
   if (!isRecordValue(request) || typeof request.query !== "string" || request.query.length === 0 || request.query.length > 256 || !request.query.isWellFormed() || /[\r\n\0]/u.test(request.query) || !Array.isArray(request.encodedPassages) || request.encodedPassages.some((item) => typeof item !== "string" || item.length === 0 || item.length % 4 !== 0 || item.length > Math.ceil((MAX_CONCEPT_CHARS + 256) * 4 / 3 * 4) + 4 || !/^[A-Za-z0-9+/]+={0,2}$/u.test(item)))
     throw new Error("Invalid concept worker input");
   const encodedPassages = request.encodedPassages.filter((item) => typeof item === "string");
@@ -497,12 +503,16 @@ async function requestFromStdin() {
     throw new Error("Invalid decoded concept worker passage");
   return { query: request.query, passages };
 }
-async function search() {
-  const request = await requestFromStdin();
+async function search(raw) {
+  const request = decodeRequest(raw);
   const directory = conceptModelDirectory();
   const cacheRoot = conceptCacheDirectory();
   const stagingRoot = process.env.SIFT_LIGHT_CONCEPT_CACHE_STAGING_DIR ?? cacheRoot;
-  await verifyConceptModel(directory);
+  if (!modelVerified) {
+    await verifyConceptModel(directory);
+    modelVerified = true;
+  }
+  const modelReused = extractor !== undefined;
   const requested = [
     { key: conceptEmbeddingKey("query", request.query), role: "query", text: request.query },
     ...request.passages.map((text) => ({
@@ -526,28 +536,27 @@ async function search() {
     env.useFSCache = false;
     env.useBrowserCache = false;
     env.localModelPath = "/";
-    emitProgress({ phase: "model-loading", ...progressContext });
-    const extractor = await pipeline("feature-extraction", directory, {
-      local_files_only: true,
-      dtype: "q8",
-      device: "cpu",
-      session_options: { intraOpNumThreads: 4, interOpNumThreads: 1 }
-    });
-    try {
-      created = await embedConceptInputs(extractor, missing, {
-        onCompletedEmbedding: async (embedding, completed, total) => {
-          try {
-            await writeConceptEmbedding(cacheRoot, embedding, stagingRoot);
-          } catch {
-            cacheWriteFailed = true;
-          }
-          emitProgress({ phase: "cache-write", completed, total, ...progressContext });
-        },
-        onBatch: (completed, total) => emitProgress({ phase: "embedding", completed, total, ...progressContext })
+    if (!extractor) {
+      emitProgress({ phase: "model-loading", ...progressContext });
+      extractor = await pipeline("feature-extraction", directory, {
+        local_files_only: true,
+        dtype: "q8",
+        device: "cpu",
+        session_options: { intraOpNumThreads: 4, interOpNumThreads: 1 }
       });
-    } finally {
-      await extractor.dispose();
+      modelLoads += 1;
     }
+    created = await embedConceptInputs(extractor, missing, {
+      onCompletedEmbedding: async (embedding, completed, total) => {
+        try {
+          await writeConceptEmbedding(cacheRoot, embedding, stagingRoot);
+        } catch {
+          cacheWriteFailed = true;
+        }
+        emitProgress({ phase: "cache-write", completed, total, ...progressContext });
+      },
+      onBatch: (completed, total) => emitProgress({ phase: "embedding", completed, total, ...progressContext })
+    });
     if (cacheWriteFailed)
       warnings.push("Concept embedding cache write failed; ranking completed but some work may repeat");
   }
@@ -575,6 +584,9 @@ async function search() {
   }
   process.stdout.write(JSON.stringify({
     type: "result",
+    id: activeRequestId,
+    modelLoads,
+    modelReused,
     scores,
     cacheHits: cached.filter(Boolean).length,
     cacheMisses: missing.length,
@@ -584,11 +596,49 @@ async function search() {
     warnings: [...new Set(warnings)],
     peakRssBytes: process.resourceUsage().maxRSS * 1024,
     modelBytes: CONCEPT_ASSETS.reduce((sum, asset) => sum + asset.bytes, 0)
-  }));
+  }) + `
+`);
+}
+async function serve() {
+  let buffer = Buffer.alloc(0);
+  for await (const chunk of process.stdin) {
+    buffer = Buffer.concat([buffer, Buffer.from(chunk)]);
+    if (buffer.length > MAX_CONCEPT_WORKER_INPUT_BYTES)
+      throw new Error("Concept request frame exceeds 64 MiB");
+    let newline = buffer.indexOf(10);
+    while (newline >= 0) {
+      const line = buffer.subarray(0, newline);
+      buffer = buffer.subarray(newline + 1);
+      const raw = JSON.parse(line.toString("utf8"));
+      if (!isRecordValue(raw) || typeof raw.id !== "string" || !/^[a-f0-9-]{36}$/u.test(raw.id))
+        throw new Error("Invalid concept request identity");
+      activeRequestId = raw.id;
+      try {
+        await search(raw);
+      } catch (error) {
+        process.stdout.write(JSON.stringify({
+          type: "error",
+          id: activeRequestId,
+          message: error instanceof Error ? error.message.slice(0, 512) : "Concept inference failed"
+        }) + `
+`);
+        return;
+      }
+      newline = buffer.indexOf(10);
+    }
+  }
+  if (buffer.length)
+    throw new Error("Incomplete concept request frame");
+}
+try {
+  if (process.argv.includes("--install-model"))
+    await installConceptModel();
+  else if (process.argv.includes("--infer"))
+    await search(await readSingleRequest());
+  else if (process.argv.includes("--serve"))
+    await serve();
+  else
+    throw new Error("Usage: sift-light-model --install-model");
+} finally {
+  await extractor?.dispose();
 }
-if (process.argv.includes("--install-model"))
-  await installConceptModel();
-else if (process.argv.includes("--infer"))
-  await search();
-else
-  throw new Error("Usage: sift-light-model --install-model");

package/plugins/sift-light/kimi.plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "sift-light",
-  "version": "1.0.3-5",
+  "version": "1.0.3-6",
   "description": "Require sift-light for conventional local searches while keeping development tools available.",
   "author": {
     "name": "baoer"
@@ -13,7 +13,7 @@
       "args": [
         "--yes",
         "--package",
-        "sift-light-runtime@npm:sift-light@1.0.3-5",
+        "sift-light-runtime@npm:sift-light@1.0.3-6",
         "sift-light-mcp",
         "--stdio"
       ],