simple-dynamsoft-mcp 7.6.1 → 7.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/package.json +4 -2
- package/scripts/eval-retrieval.mjs +64 -0
- package/scripts/lib/retrieval-eval.mjs +134 -0
- package/scripts/lib/scope-matcher.mjs +17 -0
- package/scripts/prebuild-rag-index.mjs +9 -1
- package/scripts/verify-doc-resources.mjs +2 -1
- package/src/data/repo-map.js +23 -4
- package/src/rag/config.js +4 -6
- package/src/rag/index.js +22 -2
- package/src/rag/lexical-provider.js +105 -53
- package/src/rag/search-utils.js +12 -2
- package/src/rag/vector-cache.js +77 -31
- package/src/server/resource-index/builders.js +56 -1
- package/src/server/resource-index/docs-loader.js +57 -4
- package/src/server/resource-index/shared-docs.js +161 -0
- package/src/server/resource-index.js +57 -37
- package/src/server/tools/register-index-tools.js +4 -1
package/src/rag/search-utils.js
CHANGED
|
@@ -78,9 +78,11 @@ function chunkMarkdown(text, chunkSize, chunkOverlap, maxChunks) {
|
|
|
78
78
|
const sections = [];
|
|
79
79
|
let current = { path: "", lines: [] };
|
|
80
80
|
|
|
81
|
+
// Heading-only sections (e.g. index pages whose headings are links) are
|
|
82
|
+
// kept: their heading path is the only text that makes them findable.
|
|
81
83
|
const flush = () => {
|
|
82
84
|
const body = current.lines.join(" ").replace(/\s+/g, " ").trim();
|
|
83
|
-
if (body) sections.push({ path: current.path, body });
|
|
85
|
+
if (body || current.path) sections.push({ path: current.path, body });
|
|
84
86
|
};
|
|
85
87
|
|
|
86
88
|
let inFence = false;
|
|
@@ -116,6 +118,11 @@ function chunkMarkdown(text, chunkSize, chunkOverlap, maxChunks) {
|
|
|
116
118
|
const chunks = [];
|
|
117
119
|
for (const section of sections) {
|
|
118
120
|
const prefix = section.path ? `${section.path}\n` : "";
|
|
121
|
+
if (!section.body) {
|
|
122
|
+
chunks.push(section.path);
|
|
123
|
+
if (maxChunks && chunks.length >= maxChunks) return chunks;
|
|
124
|
+
continue;
|
|
125
|
+
}
|
|
119
126
|
const pieces = chunkText(section.body, chunkSize, chunkOverlap, maxChunks);
|
|
120
127
|
for (const piece of pieces) {
|
|
121
128
|
chunks.push(`${prefix}${piece}`.trim());
|
|
@@ -248,6 +255,7 @@ function buildIndexSignature({ pkgVersion, signatureData, ragConfig }) {
|
|
|
248
255
|
dbrWebDocCount: signatureData.dbrWebDocCount,
|
|
249
256
|
dbrMobileDocCount: signatureData.dbrMobileDocCount,
|
|
250
257
|
dbrServerDocCount: signatureData.dbrServerDocCount,
|
|
258
|
+
dbrSharedDocCount: signatureData.dbrSharedDocCount,
|
|
251
259
|
dwtDocCount: signatureData.dwtDocCount,
|
|
252
260
|
ddvDocCount: signatureData.ddvDocCount,
|
|
253
261
|
versions: signatureData.versions,
|
|
@@ -257,7 +265,9 @@ function buildIndexSignature({ pkgVersion, signatureData, ragConfig }) {
|
|
|
257
265
|
maxChunksPerDoc: ragConfig.maxChunksPerDoc,
|
|
258
266
|
maxTextChars: ragConfig.maxTextChars,
|
|
259
267
|
// Bump when the chunking strategy changes so cached vector indexes rebuild.
|
|
260
|
-
chunkStrategy: "heading-aware-
|
|
268
|
+
chunkStrategy: "heading-aware-v2",
|
|
269
|
+
// Bump when doc:// URIs change shape; cached vectors are keyed by URI.
|
|
270
|
+
docUriScheme: "source-path-v1"
|
|
261
271
|
});
|
|
262
272
|
}
|
|
263
273
|
|
package/src/rag/vector-cache.js
CHANGED
|
@@ -20,6 +20,70 @@ function makeCheckpointFileName(provider, model, cacheKey) {
|
|
|
20
20
|
return `rag-${provider}-${safeModel}-${cacheKey.slice(0, 12)}.checkpoint.json`;
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
+
// Cache layout: a JSON header (`<name>.json`: cacheKey, meta, items, vector
|
|
24
|
+
// count/dimensions) plus a sibling `<name>.f32` file holding all vectors as
|
|
25
|
+
// packed little-endian float32. Serializing vectors into JSON overflowed V8's
|
|
26
|
+
// ~512MB max string length once the index grew; neither file here is built as
|
|
27
|
+
// one large string.
|
|
28
|
+
const VECTOR_FORMAT = "float32-le-v1";
|
|
29
|
+
|
|
30
|
+
function vectorFileFor(headerFile) {
|
|
31
|
+
return headerFile.replace(/\.json$/i, ".f32");
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function writeVectorFiles(headerFile, payload) {
|
|
35
|
+
const { vectors, ...header } = payload;
|
|
36
|
+
const list = Array.isArray(vectors) ? vectors : [];
|
|
37
|
+
const dimensions = list.length > 0 ? list[0].length : 0;
|
|
38
|
+
const packed = new Float32Array(list.length * dimensions);
|
|
39
|
+
list.forEach((vector, index) => {
|
|
40
|
+
if (vector.length !== dimensions) {
|
|
41
|
+
throw new Error(`Vector ${index} has ${vector.length} dimensions; expected ${dimensions}`);
|
|
42
|
+
}
|
|
43
|
+
packed.set(vector, index * dimensions);
|
|
44
|
+
});
|
|
45
|
+
// Vectors first: a header always describes a complete vector file, and a
|
|
46
|
+
// stale header next to a newer vector file fails the size check on load.
|
|
47
|
+
writeFileSync(vectorFileFor(headerFile), Buffer.from(packed.buffer, packed.byteOffset, packed.byteLength));
|
|
48
|
+
writeFileSync(headerFile, JSON.stringify({
|
|
49
|
+
...header,
|
|
50
|
+
vectorFormat: VECTOR_FORMAT,
|
|
51
|
+
vectorCount: list.length,
|
|
52
|
+
dimensions
|
|
53
|
+
}));
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// Returns the parsed header with `vectors` as Float32Array views over one
|
|
57
|
+
// buffer, or { error } describing why the files cannot be used.
|
|
58
|
+
function readVectorFiles(headerFile) {
|
|
59
|
+
const header = JSON.parse(readFileSync(headerFile, "utf8"));
|
|
60
|
+
if (!header || header.vectorFormat !== VECTOR_FORMAT || !Array.isArray(header.items)) {
|
|
61
|
+
return { error: "invalid_payload" };
|
|
62
|
+
}
|
|
63
|
+
const vectorFile = vectorFileFor(headerFile);
|
|
64
|
+
if (!existsSync(vectorFile)) {
|
|
65
|
+
return { error: "vectors_missing" };
|
|
66
|
+
}
|
|
67
|
+
const count = Number(header.vectorCount) || 0;
|
|
68
|
+
const dimensions = Number(header.dimensions) || 0;
|
|
69
|
+
const raw = readFileSync(vectorFile);
|
|
70
|
+
if (raw.byteLength !== count * dimensions * Float32Array.BYTES_PER_ELEMENT) {
|
|
71
|
+
return { error: "vectors_size_mismatch" };
|
|
72
|
+
}
|
|
73
|
+
// Copy into an aligned buffer; Buffer offsets are not guaranteed 4-byte aligned.
|
|
74
|
+
const all = new Float32Array(raw.buffer.slice(raw.byteOffset, raw.byteOffset + raw.byteLength));
|
|
75
|
+
const vectors = new Array(count);
|
|
76
|
+
for (let i = 0; i < count; i += 1) {
|
|
77
|
+
vectors[i] = all.subarray(i * dimensions, (i + 1) * dimensions);
|
|
78
|
+
}
|
|
79
|
+
return { payload: { ...header, vectors } };
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function removeVectorFiles(headerFile) {
|
|
83
|
+
rmSync(headerFile, { force: true });
|
|
84
|
+
rmSync(vectorFileFor(headerFile), { force: true });
|
|
85
|
+
}
|
|
86
|
+
|
|
23
87
|
function loadVectorIndexCache(
|
|
24
88
|
cacheFile,
|
|
25
89
|
{ cacheKey, signature, provider, model, requireSignature = false } = {}
|
|
@@ -28,12 +92,12 @@ function loadVectorIndexCache(
|
|
|
28
92
|
return { hit: false, reason: "missing", payload: null };
|
|
29
93
|
}
|
|
30
94
|
try {
|
|
31
|
-
const parsed =
|
|
32
|
-
if (
|
|
33
|
-
return { hit: false, reason:
|
|
95
|
+
const { payload: parsed, error } = readVectorFiles(cacheFile);
|
|
96
|
+
if (error) {
|
|
97
|
+
return { hit: false, reason: error, payload: null };
|
|
34
98
|
}
|
|
35
|
-
if (
|
|
36
|
-
return { hit: false, reason: "
|
|
99
|
+
if (cacheKey && parsed.cacheKey !== cacheKey) {
|
|
100
|
+
return { hit: false, reason: "cache_key_mismatch", payload: null };
|
|
37
101
|
}
|
|
38
102
|
const meta = parsed.meta || {};
|
|
39
103
|
if (provider && meta.provider && meta.provider !== provider) {
|
|
@@ -59,23 +123,7 @@ function loadVectorIndexCache(
|
|
|
59
123
|
|
|
60
124
|
function saveVectorIndexCache(cacheDir, cacheFile, payload) {
|
|
61
125
|
ensureDirectory(cacheDir);
|
|
62
|
-
|
|
63
|
-
try {
|
|
64
|
-
serialized = JSON.stringify(payload);
|
|
65
|
-
} catch (error) {
|
|
66
|
-
// JSON.stringify throws "Invalid string length" once the serialized cache
|
|
67
|
-
// exceeds V8's ~512MB max string length (reached with a large index of
|
|
68
|
-
// high-dimensional vectors). Fail with an actionable message instead.
|
|
69
|
-
if (error instanceof RangeError) {
|
|
70
|
-
const itemCount = Array.isArray(payload?.items) ? payload.items.length : "unknown";
|
|
71
|
-
throw new Error(
|
|
72
|
-
`Vector cache too large to serialize (${itemCount} items) — exceeds the JSON string limit. ` +
|
|
73
|
-
"Lower RAG_MAX_CHUNKS_PER_DOC or adopt a ceiling-proof cache format."
|
|
74
|
-
);
|
|
75
|
-
}
|
|
76
|
-
throw error;
|
|
77
|
-
}
|
|
78
|
-
writeFileSync(cacheFile, serialized);
|
|
126
|
+
writeVectorFiles(cacheFile, payload);
|
|
79
127
|
}
|
|
80
128
|
|
|
81
129
|
function loadVectorIndexCheckpoint(checkpointFile, expectedKey, expectedItems) {
|
|
@@ -83,12 +131,12 @@ function loadVectorIndexCheckpoint(checkpointFile, expectedKey, expectedItems) {
|
|
|
83
131
|
return { hit: false, reason: "missing", payload: null };
|
|
84
132
|
}
|
|
85
133
|
try {
|
|
86
|
-
const parsed =
|
|
87
|
-
if (
|
|
88
|
-
return { hit: false, reason:
|
|
134
|
+
const { payload: parsed, error } = readVectorFiles(checkpointFile);
|
|
135
|
+
if (error) {
|
|
136
|
+
return { hit: false, reason: error, payload: null };
|
|
89
137
|
}
|
|
90
|
-
if (
|
|
91
|
-
return { hit: false, reason: "
|
|
138
|
+
if (parsed.cacheKey !== expectedKey) {
|
|
139
|
+
return { hit: false, reason: "cache_key_mismatch", payload: null };
|
|
92
140
|
}
|
|
93
141
|
if (parsed.items.length !== expectedItems.length) {
|
|
94
142
|
return { hit: false, reason: "items_length_mismatch", payload: null };
|
|
@@ -109,13 +157,11 @@ function loadVectorIndexCheckpoint(checkpointFile, expectedKey, expectedItems) {
|
|
|
109
157
|
|
|
110
158
|
function saveVectorIndexCheckpoint(cacheDir, checkpointFile, payload) {
|
|
111
159
|
ensureDirectory(cacheDir);
|
|
112
|
-
|
|
160
|
+
writeVectorFiles(checkpointFile, payload);
|
|
113
161
|
}
|
|
114
162
|
|
|
115
163
|
function clearVectorIndexCheckpoint(checkpointFile) {
|
|
116
|
-
|
|
117
|
-
rmSync(checkpointFile, { force: true });
|
|
118
|
-
}
|
|
164
|
+
removeVectorFiles(checkpointFile);
|
|
119
165
|
}
|
|
120
166
|
|
|
121
167
|
function listFilesRecursive(rootDir) {
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { existsSync, readdirSync, statSync } from "node:fs";
|
|
2
2
|
import { extname, join } from "node:path";
|
|
3
3
|
import { DDV_PREFERRED_ENTRY_FILES } from "./config.js";
|
|
4
|
+
import { shortDocPath } from "./docs-loader.js";
|
|
4
5
|
import { PRODUCT_SELECTION_GUIDANCE, PUBLIC_OFFERING_PRODUCTS } from "../public-offerings.js";
|
|
5
6
|
|
|
6
7
|
const MRZ_MATCHER = /(?:\bmrz\b|machine[-\s]?readable[-\s]?zone|passport)/i;
|
|
@@ -201,6 +202,29 @@ function addPublicResourceToIndex(addResourceToIndex, entry) {
|
|
|
201
202
|
}
|
|
202
203
|
}
|
|
203
204
|
|
|
205
|
+
function encodeDocSlug(slug) {
|
|
206
|
+
return String(slug).split("/").map((segment) => encodeURIComponent(segment)).join("/");
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
// One doc:// slug per article: its short path, or (on a collision within the
|
|
210
|
+
// same URI prefix + platform, e.g. `user-guide.md` vs `user-guide/index.md`)
|
|
211
|
+
// its full source path. Shared pages keep their `slugPrefix` namespace
|
|
212
|
+
// (`dcv`, `dcv-core`) in both forms.
|
|
213
|
+
function assignDocSlugs(docs, defaultPlatform) {
|
|
214
|
+
const withPrefix = (article, slug) => (article.slugPrefix ? `${article.slugPrefix}/${slug}` : slug);
|
|
215
|
+
const keyFor = (article, slug) => `${article.platform || defaultPlatform}/${slug}`;
|
|
216
|
+
const shortSlugs = docs.map((article) => withPrefix(article, shortDocPath(article.path || article.title)));
|
|
217
|
+
const counts = new Map();
|
|
218
|
+
docs.forEach((article, i) => {
|
|
219
|
+
const key = keyFor(article, shortSlugs[i]);
|
|
220
|
+
counts.set(key, (counts.get(key) || 0) + 1);
|
|
221
|
+
});
|
|
222
|
+
return docs.map((article, i) => {
|
|
223
|
+
if (counts.get(keyFor(article, shortSlugs[i])) === 1) return shortSlugs[i];
|
|
224
|
+
return withPrefix(article, String(article.path || article.title).replace(/\\/g, "/").replace(/\.md$/i, ""));
|
|
225
|
+
});
|
|
226
|
+
}
|
|
227
|
+
|
|
204
228
|
function addMarkdownDocResources({
|
|
205
229
|
addResourceToIndex,
|
|
206
230
|
docs,
|
|
@@ -214,12 +238,18 @@ function addMarkdownDocResources({
|
|
|
214
238
|
defaultSummary,
|
|
215
239
|
baseTags
|
|
216
240
|
}) {
|
|
241
|
+
const slugs = assignDocSlugs(docs, defaultPlatform);
|
|
217
242
|
for (let i = 0; i < docs.length; i++) {
|
|
218
243
|
const article = docs[i];
|
|
219
244
|
if (!article?.title) continue;
|
|
220
|
-
const slug =
|
|
245
|
+
const slug = encodeDocSlug(slugs[i]);
|
|
221
246
|
const platform = article.platform || defaultPlatform;
|
|
222
247
|
const tags = [...baseTags, platform];
|
|
248
|
+
if (Array.isArray(article.platforms)) {
|
|
249
|
+
for (const extra of article.platforms) {
|
|
250
|
+
if (!tags.includes(extra)) tags.push(extra);
|
|
251
|
+
}
|
|
252
|
+
}
|
|
223
253
|
if (article.breadcrumb) tags.push(...article.breadcrumb.toLowerCase().split(/\s*>\s*/));
|
|
224
254
|
// Fold frontmatter keywords into tags so domain-vocabulary search hits (#142).
|
|
225
255
|
if (Array.isArray(article.keywords)) {
|
|
@@ -239,7 +269,9 @@ function addMarkdownDocResources({
|
|
|
239
269
|
product,
|
|
240
270
|
edition,
|
|
241
271
|
platform,
|
|
272
|
+
...(Array.isArray(article.platforms) ? { platforms: article.platforms } : {}),
|
|
242
273
|
version,
|
|
274
|
+
...(article.sourceVersion ? { sourceVersion: article.sourceVersion } : {}),
|
|
243
275
|
majorVersion,
|
|
244
276
|
url: article.url,
|
|
245
277
|
title: product === "mrz" || product === "mds"
|
|
@@ -601,6 +633,7 @@ function buildResourceIndex({
|
|
|
601
633
|
dbrWebDocs,
|
|
602
634
|
dbrMobileDocs,
|
|
603
635
|
dbrServerDocs,
|
|
636
|
+
dbrSharedDocs,
|
|
604
637
|
dwtDocs,
|
|
605
638
|
ddvDocs,
|
|
606
639
|
discoverDcvMobileSamples,
|
|
@@ -967,6 +1000,27 @@ function buildResourceIndex({
|
|
|
967
1000
|
baseTags: ["doc", "dbr", "server"]
|
|
968
1001
|
});
|
|
969
1002
|
|
|
1003
|
+
const sharedEditions = [
|
|
1004
|
+
["web", dbrWebVersion, "Web"],
|
|
1005
|
+
["mobile", dbrMobileVersion, "Mobile"],
|
|
1006
|
+
["server", dbrServerVersion, "Server/Desktop"]
|
|
1007
|
+
];
|
|
1008
|
+
for (const [edition, version, label] of sharedEditions) {
|
|
1009
|
+
addMarkdownDocResources({
|
|
1010
|
+
addResourceToIndex,
|
|
1011
|
+
docs: dbrSharedDocs?.[edition] || [],
|
|
1012
|
+
idPrefix: `dbr-${edition}-shared-doc`,
|
|
1013
|
+
uriPrefix: `doc://dbr/${edition}`,
|
|
1014
|
+
product: "dbr",
|
|
1015
|
+
edition,
|
|
1016
|
+
version,
|
|
1017
|
+
majorVersion: LATEST_MAJOR.dbr,
|
|
1018
|
+
defaultPlatform: edition,
|
|
1019
|
+
defaultSummary: `Shared Capture Vision API used by Dynamsoft Barcode Reader ${label}`,
|
|
1020
|
+
baseTags: ["doc", "dbr", edition, "shared", "capture-vision"]
|
|
1021
|
+
});
|
|
1022
|
+
}
|
|
1023
|
+
|
|
970
1024
|
for (const platform of getDbrMobilePlatforms()) {
|
|
971
1025
|
const samples = discoverMobileSamples(platform);
|
|
972
1026
|
for (const level of ["high-level", "low-level"]) {
|
|
@@ -1155,6 +1209,7 @@ function buildResourceIndex({
|
|
|
1155
1209
|
}
|
|
1156
1210
|
|
|
1157
1211
|
export {
|
|
1212
|
+
assignDocSlugs,
|
|
1158
1213
|
buildIndexData,
|
|
1159
1214
|
buildResourceIndex
|
|
1160
1215
|
};
|
|
@@ -98,9 +98,56 @@ function markdownPathToUrl(baseUrl, relativePath) {
|
|
|
98
98
|
return `${baseUrl}${normalized}`;
|
|
99
99
|
}
|
|
100
100
|
|
|
101
|
-
|
|
101
|
+
// Path used in doc:// URIs. The platform is already a URI segment, so the
|
|
102
|
+
// `programming/<lang>/` prefix is dropped, and a directory's index page is
|
|
103
|
+
// addressed by the directory.
|
|
104
|
+
function shortDocPath(relativePath) {
|
|
105
|
+
const withoutExt = String(relativePath || "").replace(/\\/g, "/").replace(/\.md$/i, "");
|
|
106
|
+
const withoutPlatform = withoutExt.replace(/^programming\/[^/]+\//, "");
|
|
107
|
+
const withoutIndex = withoutPlatform.replace(/(^|\/)index$/, "");
|
|
108
|
+
return withoutIndex || "index";
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
const SITE_ORIGIN = "https://www.dynamsoft.com";
|
|
112
|
+
|
|
113
|
+
// Top-level `key: value` pairs from a Jekyll _config.yml. Only path/URL values
|
|
114
|
+
// are kept: they are what docs interpolate as `{{ site.key }}` link prefixes.
|
|
115
|
+
function loadSiteVariables(siteRoot) {
|
|
116
|
+
const vars = new Map();
|
|
117
|
+
const configPath = join(siteRoot || "", "_config.yml");
|
|
118
|
+
if (!siteRoot || !existsSync(configPath)) return vars;
|
|
119
|
+
for (const line of readFileSync(configPath, "utf8").split(/\r?\n/)) {
|
|
120
|
+
const match = line.match(/^([A-Za-z0-9_-]+):\s*["']?([^"'#\s]+)["']?\s*(?:#.*)?$/);
|
|
121
|
+
if (!match) continue;
|
|
122
|
+
const [, key, value] = match;
|
|
123
|
+
if (value.startsWith("/")) {
|
|
124
|
+
vars.set(key, `${SITE_ORIGIN}${value}`);
|
|
125
|
+
} else if (/^https?:\/\//i.test(value)) {
|
|
126
|
+
vars.set(key, value);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
return vars;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// Replace `{{ site.key }}` with the absolute URL from _config.yml so links in
|
|
133
|
+
// returned docs (and the shared-doc link map) point at the real public pages.
|
|
134
|
+
// Unknown keys are left untouched.
|
|
135
|
+
function resolveSiteLinks(text, siteVars) {
|
|
136
|
+
if (!text || !siteVars || siteVars.size === 0) return text;
|
|
137
|
+
return String(text).replace(/\{\{\s*site\.([A-Za-z0-9_-]+)\s*\}\}/g, (whole, key) => siteVars.get(key) ?? whole);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
function loadMarkdownDocs({
|
|
141
|
+
rootDir,
|
|
142
|
+
urlBase,
|
|
143
|
+
siteRoot = rootDir,
|
|
144
|
+
includeDirNames = [],
|
|
145
|
+
excludeDirs = [],
|
|
146
|
+
excludeFiles = []
|
|
147
|
+
}) {
|
|
102
148
|
if (!existsSync(rootDir)) return { articles: [] };
|
|
103
149
|
|
|
150
|
+
const siteVars = loadSiteVariables(siteRoot);
|
|
104
151
|
const files = collectMarkdownFiles(rootDir, { includeDirNames, excludeDirs, excludeFiles });
|
|
105
152
|
const articles = [];
|
|
106
153
|
|
|
@@ -108,6 +155,9 @@ function loadMarkdownDocs({ rootDir, urlBase, includeDirNames = [], excludeDirs
|
|
|
108
155
|
const relativePath = relative(rootDir, filePath).replace(/\\/g, "/");
|
|
109
156
|
const raw = readFileSync(filePath, "utf8");
|
|
110
157
|
const parsed = parseFrontMatter(raw);
|
|
158
|
+
// Jekyll redirect stubs have no content of their own; indexing them yields
|
|
159
|
+
// empty hits whose URL only redirects elsewhere.
|
|
160
|
+
if (parsed.meta.redirect_to) continue;
|
|
111
161
|
const title = parsed.meta.title || getHeadingTitle(parsed.body) || formatSegmentLabel(relativePath.replace(/\.md$/i, "").split("/").pop());
|
|
112
162
|
if (!title) continue;
|
|
113
163
|
const breadcrumb = parsed.meta.breadcrumbText || buildBreadcrumbFromPath(relativePath) || title;
|
|
@@ -121,7 +171,7 @@ function loadMarkdownDocs({ rootDir, urlBase, includeDirNames = [], excludeDirs
|
|
|
121
171
|
articles.push({
|
|
122
172
|
title,
|
|
123
173
|
url: markdownPathToUrl(urlBase, relativePath),
|
|
124
|
-
content: parsed.body.trim(),
|
|
174
|
+
content: resolveSiteLinks(parsed.body.trim(), siteVars),
|
|
125
175
|
breadcrumb,
|
|
126
176
|
description,
|
|
127
177
|
keywords,
|
|
@@ -129,9 +179,12 @@ function loadMarkdownDocs({ rootDir, urlBase, includeDirNames = [], excludeDirs
|
|
|
129
179
|
});
|
|
130
180
|
}
|
|
131
181
|
|
|
132
|
-
return { articles };
|
|
182
|
+
return { articles, siteVars };
|
|
133
183
|
}
|
|
134
184
|
|
|
135
185
|
export {
|
|
136
|
-
loadMarkdownDocs
|
|
186
|
+
loadMarkdownDocs,
|
|
187
|
+
loadSiteVariables,
|
|
188
|
+
resolveSiteLinks,
|
|
189
|
+
shortDocPath
|
|
137
190
|
};
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
import { existsSync, readFileSync, readdirSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { resolveSiteLinks } from "./docs-loader.js";
|
|
4
|
+
|
|
5
|
+
// DBR's public API is built on shared Capture Vision (DCV) modules such as
|
|
6
|
+
// CaptureVisionRouter, whose reference pages live only in the DCV docs. DBR's
|
|
7
|
+
// own docs and navigation sidebars link to exactly the DCV pages DBR relies on,
|
|
8
|
+
// so those links are the dependency map: only linked pages are indexed under
|
|
9
|
+
// DBR, never the whole DCV corpus.
|
|
10
|
+
|
|
11
|
+
const DCV_CURRENT_DOCS_PREFIX = "https://www.dynamsoft.com/capture-vision/docs/";
|
|
12
|
+
|
|
13
|
+
// Sidebar include `sidelist-programming/programming-<segment>.html` scopes its
|
|
14
|
+
// links to one DBR platform; other includes apply to the whole edition.
|
|
15
|
+
const SIDEBAR_PLATFORM_BY_SEGMENT = {
|
|
16
|
+
javascript: "web",
|
|
17
|
+
android: "android",
|
|
18
|
+
oc: "ios",
|
|
19
|
+
ios: "ios",
|
|
20
|
+
flutter: "flutter",
|
|
21
|
+
maui: "maui",
|
|
22
|
+
"react-native": "react-native",
|
|
23
|
+
cpp: "cpp",
|
|
24
|
+
dotnet: "dotnet",
|
|
25
|
+
java: "java",
|
|
26
|
+
python: "python",
|
|
27
|
+
nodejs: "nodejs"
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
function normalizeDocUrl(url) {
|
|
31
|
+
// Some DBR links contain `//` inside the path (e.g. `api-reference//core/`).
|
|
32
|
+
let normalized = String(url || "").split("#")[0].split("?")[0].replace(/([^:])\/{2,}/g, "$1/");
|
|
33
|
+
if (!normalized) return "";
|
|
34
|
+
if (normalized.endsWith("/")) return `${normalized}index.html`;
|
|
35
|
+
const lastSegment = normalized.slice(normalized.lastIndexOf("/") + 1);
|
|
36
|
+
if (!lastSegment.includes(".")) normalized = `${normalized}.html`;
|
|
37
|
+
return normalized;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function extractDcvLinks(text) {
|
|
41
|
+
const links = new Set();
|
|
42
|
+
const pattern = /https:\/\/www\.dynamsoft\.com\/capture-vision\/docs\/[^\s)"'<>\]`]+/g;
|
|
43
|
+
for (const match of String(text || "").matchAll(pattern)) {
|
|
44
|
+
const normalized = normalizeDocUrl(match[0]);
|
|
45
|
+
if (normalized.startsWith(DCV_CURRENT_DOCS_PREFIX)) links.add(normalized);
|
|
46
|
+
}
|
|
47
|
+
return links;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function listIncludeFiles(dir) {
|
|
51
|
+
const files = [];
|
|
52
|
+
if (!existsSync(dir)) return files;
|
|
53
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
54
|
+
const fullPath = join(dir, entry.name);
|
|
55
|
+
if (entry.isDirectory()) {
|
|
56
|
+
files.push(...listIncludeFiles(fullPath));
|
|
57
|
+
} else if (/\.(html|md)$/i.test(entry.name)) {
|
|
58
|
+
files.push({ path: fullPath, name: entry.name });
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
return files;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function sidebarPlatform(fileName) {
|
|
65
|
+
const match = String(fileName).match(/^programming-(.+)\.html$/i);
|
|
66
|
+
return match ? SIDEBAR_PLATFORM_BY_SEGMENT[match[1].toLowerCase()] || "" : "";
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// url -> Set of DBR platforms whose docs link to it ("*" = whole edition).
|
|
70
|
+
function buildDcvLinkMap({ dbrArticles, dbrRoot, siteVars }) {
|
|
71
|
+
const linkMap = new Map();
|
|
72
|
+
const addLinks = (links, platform) => {
|
|
73
|
+
for (const url of links) {
|
|
74
|
+
if (!linkMap.has(url)) linkMap.set(url, new Set());
|
|
75
|
+
linkMap.get(url).add(platform);
|
|
76
|
+
}
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
for (const article of dbrArticles || []) {
|
|
80
|
+
addLinks(extractDcvLinks(article.content), article.platform || "*");
|
|
81
|
+
}
|
|
82
|
+
for (const file of listIncludeFiles(join(dbrRoot || "", "_includes"))) {
|
|
83
|
+
const text = resolveSiteLinks(readFileSync(file.path, "utf8"), siteVars);
|
|
84
|
+
addLinks(extractDcvLinks(text), sidebarPlatform(file.name) || "*");
|
|
85
|
+
}
|
|
86
|
+
return linkMap;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function parseVersionParts(version) {
|
|
90
|
+
return String(version || "").split(".").map((part) => Number.parseInt(part, 10));
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// DBR and DCV ship in lockstep: 11.6.3200 pairs with 3.6.3200. Edition pages
|
|
94
|
+
// must match minor and build; core reference pages are edition-neutral and
|
|
95
|
+
// only need the same minor.
|
|
96
|
+
function isCompatibleSharedVersion(dbrVersion, dcvVersion, { requireBuild }) {
|
|
97
|
+
const [, dbrMinor, dbrBuild] = parseVersionParts(dbrVersion);
|
|
98
|
+
const [, dcvMinor, dcvBuild] = parseVersionParts(dcvVersion);
|
|
99
|
+
if (!Number.isFinite(dbrMinor) || dbrMinor !== dcvMinor) return false;
|
|
100
|
+
if (requireBuild && dbrBuild !== dcvBuild) return false;
|
|
101
|
+
return true;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Select DCV pages linked from one DBR edition.
|
|
106
|
+
*
|
|
107
|
+
* @param {object} options
|
|
108
|
+
* @param {Array} options.dbrArticles DBR edition articles (with platform).
|
|
109
|
+
* @param {string} options.dbrRoot DBR docs repo root (for `_includes`).
|
|
110
|
+
* @param {Map} options.siteVars DBR repo `{{ site.* }}` variables.
|
|
111
|
+
* @param {string} options.dbrVersion DBR edition version.
|
|
112
|
+
* @param {Array<{articles: Array, version: string, editionSpecific: boolean, slugPrefix: string}>} options.sources
|
|
113
|
+
* @param {Function} [options.onSkip] Called with details when a source is version-incompatible.
|
|
114
|
+
* @returns {Array} Articles with `slugPrefix` (doc:// namespace), `platform`, `platforms`, `sourceVersion`.
|
|
115
|
+
*/
|
|
116
|
+
function selectSharedDcvDocs({ dbrArticles, dbrRoot, siteVars, dbrVersion, sources, onSkip }) {
|
|
117
|
+
const linkMap = buildDcvLinkMap({ dbrArticles, dbrRoot, siteVars });
|
|
118
|
+
const editionPlatforms = [...new Set((dbrArticles || []).map((article) => article.platform).filter(Boolean))].sort();
|
|
119
|
+
const selected = [];
|
|
120
|
+
|
|
121
|
+
for (const source of sources) {
|
|
122
|
+
if (!isCompatibleSharedVersion(dbrVersion, source.version, { requireBuild: source.editionSpecific })) {
|
|
123
|
+
onSkip?.({ dbrVersion, dcvVersion: source.version, slugPrefix: source.slugPrefix });
|
|
124
|
+
continue;
|
|
125
|
+
}
|
|
126
|
+
for (const article of source.articles || []) {
|
|
127
|
+
const linkedFrom = linkMap.get(normalizeDocUrl(article.url));
|
|
128
|
+
if (!linkedFrom) continue;
|
|
129
|
+
|
|
130
|
+
let platforms;
|
|
131
|
+
if (source.editionSpecific) {
|
|
132
|
+
// An edition page belongs to one platform (its programming/<lang>/ dir).
|
|
133
|
+
if (!editionPlatforms.includes(article.platform) && editionPlatforms.length > 0) continue;
|
|
134
|
+
platforms = [article.platform];
|
|
135
|
+
} else {
|
|
136
|
+
platforms = linkedFrom.has("*")
|
|
137
|
+
? editionPlatforms
|
|
138
|
+
: [...linkedFrom].filter((platform) => editionPlatforms.includes(platform)).sort();
|
|
139
|
+
}
|
|
140
|
+
if (platforms.length === 0) continue;
|
|
141
|
+
|
|
142
|
+
selected.push({
|
|
143
|
+
...article,
|
|
144
|
+
slugPrefix: source.slugPrefix,
|
|
145
|
+
platform: platforms.length === 1 ? platforms[0] : (source.defaultPlatform || platforms[0]),
|
|
146
|
+
platforms,
|
|
147
|
+
sourceVersion: source.version
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
return selected;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
export {
|
|
156
|
+
buildDcvLinkMap,
|
|
157
|
+
extractDcvLinks,
|
|
158
|
+
isCompatibleSharedVersion,
|
|
159
|
+
normalizeDocUrl,
|
|
160
|
+
selectSharedDcvDocs
|
|
161
|
+
};
|