verifact 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +94 -2
- package/VERIFACT-DESIGN.md +337 -0
- package/bin/verifact.js +463 -0
- package/package.json +49 -4
- package/src/config.js +198 -0
- package/src/core/artifacts.js +275 -0
- package/src/core/cache.js +87 -0
- package/src/core/claims.js +329 -0
- package/src/core/crossdoc.js +97 -0
- package/src/core/dispositions.js +292 -0
- package/src/core/lint.js +111 -0
- package/src/core/normalize.js +178 -0
- package/src/core/resolve.js +307 -0
- package/src/core/stable-key.js +97 -0
- package/src/core/walk.js +99 -0
- package/src/engine.js +467 -0
- package/src/errors.js +35 -0
- package/src/history.js +91 -0
- package/src/logger.js +29 -0
- package/src/pooled-scan.js +109 -0
- package/src/server/api.js +167 -0
- package/src/server/www/app.js +203 -0
- package/src/server/www/index.html +83 -0
- package/src/server/www/style.css +101 -0
- package/src/worker.js +35 -0
- package/verifact.json.example +92 -0
package/src/config.js
ADDED
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
const fs = require('fs');
|
|
5
|
+
const path = require('path');
|
|
6
|
+
const { ConfigError } = require('./errors');
|
|
7
|
+
const { dekey } = require('./core/normalize');
|
|
8
|
+
const { DEFAULT_EXCLUDE } = require('./core/walk');
|
|
9
|
+
|
|
10
|
+
const DEFAULT_MODULES = [
|
|
11
|
+
'ML-KEM-768', 'ML-KEM-1024', 'ML-KEM', 'ML-DSA', 'SLH-DSA',
|
|
12
|
+
'SM2', 'SM3', 'SM4', 'SM4-GCM', 'HMAC-SM3', 'P-256/ECDH',
|
|
13
|
+
'FPGA NTT', 'FPGA v5', 'NTT', 'VWZ', 'LookingGlass',
|
|
14
|
+
'TLS 1.3 Hybrid', 'Path C-2', 'C-2', 'OPK', 'TLA+', 'Double-Ratchet',
|
|
15
|
+
'Jasmin', 'liboqs', 'ADLA', 'TVLA'
|
|
16
|
+
// 注意:CARS / CBOM / SBOM 是评估框架不是模块,不要放进这张表——
|
|
17
|
+
// 否则 cars-scorecard.json 这类文件名会被当成模块,凭空制造「跨模块绑定」。
|
|
18
|
+
];
|
|
19
|
+
|
|
20
|
+
const DEFAULTS = {
|
|
21
|
+
docs: {
|
|
22
|
+
roots: [],
|
|
23
|
+
include: ['**/*.md'],
|
|
24
|
+
exclude: DEFAULT_EXCLUDE.concat(['**/CHANGELOG.md']),
|
|
25
|
+
maxFileBytes: 2 * 1024 * 1024,
|
|
26
|
+
maxFiles: 4000,
|
|
27
|
+
maxDepth: 12
|
|
28
|
+
},
|
|
29
|
+
artifacts: {
|
|
30
|
+
roots: [],
|
|
31
|
+
// 只登记「文件是否存在」的目录:TSR/TSQ/SHA256 这类二进制凭证不解析内容,
|
|
32
|
+
// 但文档常常引用它们的编号(TSR lg-069),存在性本身就是可核验的事实。
|
|
33
|
+
presenceRoots: [],
|
|
34
|
+
include: ['**/*.json', '**/*.md', '**/*.log', '**/*.csv', '**/*.rpt', '**/*.twr'],
|
|
35
|
+
exclude: DEFAULT_EXCLUDE.concat(['**/*.min.json']),
|
|
36
|
+
maxFileBytes: 8 * 1024 * 1024,
|
|
37
|
+
maxFiles: 6000,
|
|
38
|
+
maxDepth: 12
|
|
39
|
+
},
|
|
40
|
+
resolve: { minScore: 0.45, relTolerance: 0.005, absTolerance: 0, locTolerance: 0.05 },
|
|
41
|
+
crossDoc: { enabled: true, warnRatio: 10 },
|
|
42
|
+
lint: { enabled: true, disable: [], includeCodeBlocks: false },
|
|
43
|
+
modules: DEFAULT_MODULES,
|
|
44
|
+
bindings: [],
|
|
45
|
+
ignore: [],
|
|
46
|
+
deny: [],
|
|
47
|
+
// 哈希存证目录:文件本身被时间戳盖过,插一行注释就改变哈希、破坏存证。
|
|
48
|
+
// 这类目录不能靠 disposition(那要改文件),只能靠配置登记。
|
|
49
|
+
sealed: [],
|
|
50
|
+
gate: { failOn: 'drift' },
|
|
51
|
+
cache: { enabled: true, dir: '.verifact', maxHistory: 200 },
|
|
52
|
+
server: { host: '127.0.0.1', port: 8787 }
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
/** 去掉值为 undefined 的键:deepMerge 遇到显式 undefined 会把默认值覆盖掉。 */
|
|
56
|
+
function stripUndefined(obj) {
|
|
57
|
+
const out = {};
|
|
58
|
+
for (const k of Object.keys(obj)) if (obj[k] !== undefined) out[k] = obj[k];
|
|
59
|
+
return out;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function isPlainObject(v) {
|
|
63
|
+
return v && typeof v === 'object' && !Array.isArray(v);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function deepMerge(base, over) {
|
|
67
|
+
if (!isPlainObject(over)) return over == null ? base : over;
|
|
68
|
+
const out = Object.assign({}, base);
|
|
69
|
+
for (const k of Object.keys(over)) {
|
|
70
|
+
out[k] = isPlainObject(base[k]) || Array.isArray(base[k])
|
|
71
|
+
? (Array.isArray(over[k]) ? over[k] : deepMerge(base[k] || {}, over[k]))
|
|
72
|
+
: over[k];
|
|
73
|
+
}
|
|
74
|
+
return out;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function buildModules(list) {
|
|
78
|
+
return (list || []).map((name) => ({ name, dekey: dekey(name) })).filter((m) => m.dekey.length >= 2);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function findConfigFile(explicit, cwd) {
|
|
82
|
+
if (explicit) {
|
|
83
|
+
const p = path.resolve(cwd, explicit);
|
|
84
|
+
if (!fs.existsSync(p)) throw new ConfigError('config file not found', { path: p });
|
|
85
|
+
return p;
|
|
86
|
+
}
|
|
87
|
+
for (const name of ['verifact.json', '.verifact.json', 'verifact.config.json']) {
|
|
88
|
+
const p = path.join(cwd, name);
|
|
89
|
+
if (fs.existsSync(p)) return p;
|
|
90
|
+
}
|
|
91
|
+
return null;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* 集中配置加载 + 校验。启动即失败,不让错误配置跑到半路。
|
|
96
|
+
*/
|
|
97
|
+
function loadConfig(opts) {
|
|
98
|
+
const options = opts || {};
|
|
99
|
+
const cwd = path.resolve(options.cwd || process.cwd());
|
|
100
|
+
const file = findConfigFile(options.configPath, cwd);
|
|
101
|
+
let fileCfg = {};
|
|
102
|
+
if (file) {
|
|
103
|
+
try {
|
|
104
|
+
fileCfg = JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
105
|
+
} catch (e) {
|
|
106
|
+
throw new ConfigError('config file is not valid JSON', { path: file, detail: e.message });
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// 环境变量覆盖(服务端运行场景)
|
|
111
|
+
const envCfg = {
|
|
112
|
+
server: stripUndefined({
|
|
113
|
+
host: process.env.VERIFACT_HOST,
|
|
114
|
+
port: process.env.VERIFACT_PORT ? Number(process.env.VERIFACT_PORT) : undefined
|
|
115
|
+
})
|
|
116
|
+
};
|
|
117
|
+
if (process.env.VERIFACT_CACHE === '0') envCfg.cache = { enabled: false };
|
|
118
|
+
|
|
119
|
+
const merged = deepMerge(deepMerge(DEFAULTS, fileCfg), deepMerge(envCfg, options.overrides || {}));
|
|
120
|
+
merged.__cwd = cwd;
|
|
121
|
+
merged.__configFile = file;
|
|
122
|
+
// 配置里的相对 root 以「配置文件所在目录」为基准:换一个 cwd 跑,root 不会集体落空。
|
|
123
|
+
// root 落空会被静默跳过 → 产出物索引为空 → 声明全体退化成 unbound,drift 门禁假绿。
|
|
124
|
+
// 没有配置文件时(纯 overrides / 测试)退回 cwd,保持旧行为。
|
|
125
|
+
merged.__baseDir = file ? path.dirname(file) : cwd;
|
|
126
|
+
|
|
127
|
+
// --- 校验 ---
|
|
128
|
+
const errs = [];
|
|
129
|
+
if (!Array.isArray(merged.docs.roots) || !merged.docs.roots.length) {
|
|
130
|
+
errs.push('docs.roots 不能为空');
|
|
131
|
+
}
|
|
132
|
+
if (!Array.isArray(merged.artifacts.roots)) {
|
|
133
|
+
errs.push('artifacts.roots 必须是数组');
|
|
134
|
+
}
|
|
135
|
+
if (!Array.isArray(merged.deny)) {
|
|
136
|
+
errs.push('deny 必须是数组');
|
|
137
|
+
}
|
|
138
|
+
if (!Array.isArray(merged.ignore)) {
|
|
139
|
+
errs.push('ignore 必须是数组');
|
|
140
|
+
} else {
|
|
141
|
+
// 没有 why 的豁免就是洗白:读者看不到「为什么这条不该核」,只能选择相信。
|
|
142
|
+
// 配置层就把它当致命错——不降级、不补默认值。
|
|
143
|
+
const PLACEHOLDER = /^(todo|tbd|n\/a|na|none|-+|\.+|x+|修绑定)$/i;
|
|
144
|
+
for (const r of merged.ignore) {
|
|
145
|
+
if (!r || typeof r !== 'object' || Array.isArray(r)) {
|
|
146
|
+
errs.push('ignore 的每一项必须是规则对象,实际=' + JSON.stringify(r));
|
|
147
|
+
continue;
|
|
148
|
+
}
|
|
149
|
+
if (r.note !== undefined && r.why === undefined) {
|
|
150
|
+
errs.push('ignore 条目仍用 note 而没有 why(' + JSON.stringify(r.key || r.file) + '):字段已改名,请写 why 说明理由');
|
|
151
|
+
continue;
|
|
152
|
+
}
|
|
153
|
+
if (typeof r.why !== 'string' || r.why.trim().length < 12) {
|
|
154
|
+
errs.push('ignore 条目缺少可读的 why(至少 12 字,说明为什么这条不该核):' + JSON.stringify(r.key || r.file));
|
|
155
|
+
continue;
|
|
156
|
+
}
|
|
157
|
+
if (PLACEHOLDER.test(r.why.trim())) {
|
|
158
|
+
errs.push('ignore 的 why 是占位符(' + r.why.trim() + '):' + JSON.stringify(r.key || r.file));
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
if (!Array.isArray(merged.sealed)) {
|
|
163
|
+
errs.push('sealed 必须是数组');
|
|
164
|
+
} else {
|
|
165
|
+
// sealed 只接受「路径 glob 字符串」。写成规则对象时静默不生效,
|
|
166
|
+
// 会让人以为存证目录已经受保护(假安全)——所以一律报错。
|
|
167
|
+
for (const s of merged.sealed) {
|
|
168
|
+
if (typeof s !== 'string' || !s.trim()) errs.push('sealed 的每一项必须是非空路径 glob 字符串,实际=' + JSON.stringify(s));
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
if (!(merged.resolve.minScore >= 0 && merged.resolve.minScore <= 1)) {
|
|
172
|
+
errs.push('resolve.minScore 必须在 [0,1]');
|
|
173
|
+
}
|
|
174
|
+
if (Number.isNaN(merged.server.port)) errs.push('server.port 必须是数字');
|
|
175
|
+
if (errs.length) throw new ConfigError('配置校验未通过', { problems: errs, file: merged.__configFile });
|
|
176
|
+
|
|
177
|
+
for (const r of merged.docs.roots.concat(merged.artifacts.roots)) {
|
|
178
|
+
if (typeof r !== 'string') throw new ConfigError('root 必须是字符串路径', { root: r });
|
|
179
|
+
}
|
|
180
|
+
return merged;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/** root 既可以是目录也可以是单个文件,这里分开处理。 */
|
|
184
|
+
function partitionRoots(roots, cwd) {
|
|
185
|
+
const dirs = [];
|
|
186
|
+
const files = [];
|
|
187
|
+
const missing = [];
|
|
188
|
+
for (const r of roots) {
|
|
189
|
+
const abs = path.resolve(cwd, r);
|
|
190
|
+
let st = null;
|
|
191
|
+
try { st = fs.statSync(abs); } catch (_) { missing.push(r); continue; }
|
|
192
|
+
if (st.isDirectory()) dirs.push({ input: r, abs });
|
|
193
|
+
else if (st.isFile()) files.push({ input: r, abs, rel: path.basename(abs) });
|
|
194
|
+
}
|
|
195
|
+
return { dirs, files, missing };
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
module.exports = { loadConfig, buildModules, partitionRoots, DEFAULTS, DEFAULT_MODULES, deepMerge };
|
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
const fs = require('fs');
|
|
5
|
+
const path = require('path');
|
|
6
|
+
const { iterate } = require('./walk');
|
|
7
|
+
const { tokens, valueFromString, squash } = require('./normalize');
|
|
8
|
+
const { extractClaims } = require('./claims');
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* 产出物收集器:把 JSON / 文本报告 / 日志里的数字摊平成
|
|
12
|
+
* `{source, path, keyTokens, primaryToken, module, value}` 条目,供解析器比对。
|
|
13
|
+
*
|
|
14
|
+
* 关键设计:JSON 的字符串叶子会再走一次声明抽取器,于是
|
|
15
|
+
* `"verification": "KAT 10000/10000 Noble cross-validated, TVLA 3/3 PASS"`
|
|
16
|
+
* 会变成两条独立可比对条目 `…verification#kat` 与 `…verification#tvla`。
|
|
17
|
+
* 这样文档里写的「KAT 10,000/10,000」就能直接对上,不需要人工写绑定表。
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
const TEXT_EXT = new Set(['.md', '.log', '.txt', '.csv', '.rpt', '.twr', '.out', '.yaml', '.yml']);
|
|
21
|
+
const PRESENCE_EXT = new Set(['.tsr', '.pdf', '.json', '.md', '.bin', '.hex', '.bit']);
|
|
22
|
+
// 代码文件:只数行数(供「N 行」声明核验),不抽取声明
|
|
23
|
+
const CODE_EXT = new Set(['.js', '.mjs', '.cjs', '.ts', '.py', '.c', '.h', '.cc', '.cpp', '.rs', '.go', '.java']);
|
|
24
|
+
const CODE_LINE_MAX_BYTES = 4 * 1024 * 1024;
|
|
25
|
+
const MAX_LEAVES_PER_FILE = 4000;
|
|
26
|
+
|
|
27
|
+
function extOf(rel) {
|
|
28
|
+
const i = rel.lastIndexOf('.');
|
|
29
|
+
return i < 0 ? '' : rel.slice(i).toLowerCase();
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function moduleOf(str, modules) {
|
|
33
|
+
const s = squash(str);
|
|
34
|
+
let best = null;
|
|
35
|
+
for (const m of modules) {
|
|
36
|
+
if (m.dekey && s.indexOf(m.dekey) >= 0) {
|
|
37
|
+
if (!best || m.name.length > best.length) best = m.name;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
return best;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function leafEntry(source, pathStr, value, raw, modules) {
|
|
44
|
+
const combo = source + ' ' + pathStr;
|
|
45
|
+
const keyTokens = tokens(pathStr.replace(/#/g, ' '));
|
|
46
|
+
return {
|
|
47
|
+
source,
|
|
48
|
+
path: pathStr,
|
|
49
|
+
keyTokens,
|
|
50
|
+
primaryToken: keyTokens.length ? keyTokens[keyTokens.length - 1] : null,
|
|
51
|
+
module: moduleOf(combo, modules),
|
|
52
|
+
value,
|
|
53
|
+
raw: raw == null ? null : String(raw).slice(0, 300)
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function walkJson(node, basePath, sink, modules, source) {
|
|
58
|
+
const seen = [];
|
|
59
|
+
|
|
60
|
+
(function rec(val, p) {
|
|
61
|
+
if (sink.length >= MAX_LEAVES_PER_FILE) return;
|
|
62
|
+
if (val == null) { sink.push(leafEntry(source, p, null, null, modules)); return; }
|
|
63
|
+
const t = typeof val;
|
|
64
|
+
if (t === 'number' || t === 'boolean') {
|
|
65
|
+
sink.push(leafEntry(source, p, valueFromString(String(val)), val, modules));
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
if (t === 'string') {
|
|
69
|
+
const v = valueFromString(val);
|
|
70
|
+
if (v) sink.push(leafEntry(source, p, v, val, modules));
|
|
71
|
+
// 字符串里嵌着多个度量时,逐个拆出来
|
|
72
|
+
if (/[\d]/.test(val)) {
|
|
73
|
+
const sub = extractClaims(val, source, { modules });
|
|
74
|
+
for (const c of sub) {
|
|
75
|
+
sink.push(leafEntry(source, p + '#' + c.key, c.value, c.value.raw, modules));
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
return;
|
|
79
|
+
}
|
|
80
|
+
if (Array.isArray(val)) {
|
|
81
|
+
const cap = Math.min(val.length, 200);
|
|
82
|
+
for (let i = 0; i < cap; i++) rec(val[i], p + '[' + i + ']');
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
if (t === 'object') {
|
|
86
|
+
if (seen.indexOf(val) >= 0) return; // 环保护
|
|
87
|
+
seen.push(val);
|
|
88
|
+
for (const k of Object.keys(val)) rec(val[k], p ? p + '.' + k : k);
|
|
89
|
+
}
|
|
90
|
+
})(node, basePath);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function countLines(text) {
|
|
94
|
+
// 与 wc -l 一致:统计换行符数;末行无换行时补 1
|
|
95
|
+
if (!text.length) return 0;
|
|
96
|
+
let n = 0;
|
|
97
|
+
for (let i = 0; i < text.length; i++) if (text.charCodeAt(i) === 10) n++;
|
|
98
|
+
return text.charCodeAt(text.length - 1) === 10 ? n : n + 1;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function collectFile(file, roots, modules, maxBytes, fileLines) {
|
|
102
|
+
const rel = roots.relLabel(file);
|
|
103
|
+
const ext = extOf(rel);
|
|
104
|
+
const out = [];
|
|
105
|
+
try {
|
|
106
|
+
if (ext === '.json') {
|
|
107
|
+
const text = fs.readFileSync(file, 'utf8');
|
|
108
|
+
if (maxBytes && Buffer.byteLength(text) > maxBytes) return out;
|
|
109
|
+
let data;
|
|
110
|
+
try { data = JSON.parse(text); } catch (_) { return out; }
|
|
111
|
+
walkJson(data, '', out, modules, rel);
|
|
112
|
+
return out;
|
|
113
|
+
}
|
|
114
|
+
if (TEXT_EXT.has(ext)) {
|
|
115
|
+
const text = fs.readFileSync(file, 'utf8');
|
|
116
|
+
if (maxBytes && Buffer.byteLength(text) > maxBytes) return out;
|
|
117
|
+
if (fileLines) fileLines.set(rel, countLines(text));
|
|
118
|
+
const claims = extractClaims(text, rel, { modules });
|
|
119
|
+
for (const c of claims) {
|
|
120
|
+
out.push({
|
|
121
|
+
source: rel,
|
|
122
|
+
path: rel + '#L' + c.line + '#' + c.key,
|
|
123
|
+
keyTokens: c.keyTokens,
|
|
124
|
+
primaryToken: c.primaryToken,
|
|
125
|
+
module: c.module || moduleOf(rel, modules),
|
|
126
|
+
value: c.value,
|
|
127
|
+
raw: c.sentence
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
return out;
|
|
131
|
+
}
|
|
132
|
+
} catch (_) {
|
|
133
|
+
return out;
|
|
134
|
+
}
|
|
135
|
+
return out;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* @returns {{entries:Array, presence:Array<string>, scanned:number, skipped:number}}
|
|
140
|
+
*/
|
|
141
|
+
function collectArtifacts(cfg, modules) {
|
|
142
|
+
const entries = [];
|
|
143
|
+
const presence = [];
|
|
144
|
+
const fileLines = new Map(); // 文本文件 rel → 行数(供「N 行」声明核验)
|
|
145
|
+
let scanned = 0;
|
|
146
|
+
let skipped = 0;
|
|
147
|
+
const maxBytes = cfg.artifacts.maxFileBytes;
|
|
148
|
+
const seen = new Set(); // 根目录互相嵌套时去重,否则会产生「双候选→ambiguous」
|
|
149
|
+
const seenPresence = new Set();
|
|
150
|
+
const relRoot = path;
|
|
151
|
+
|
|
152
|
+
const matchInclude = (rel) => {
|
|
153
|
+
const ext = extOf(rel);
|
|
154
|
+
return ext === '.json' || TEXT_EXT.has(ext);
|
|
155
|
+
};
|
|
156
|
+
|
|
157
|
+
const handleDir = (root) => {
|
|
158
|
+
const rootLabel = path.basename(root) || root;
|
|
159
|
+
for (const f of iterate(root, {
|
|
160
|
+
exclude: cfg.artifacts.exclude,
|
|
161
|
+
matchInclude,
|
|
162
|
+
maxFileBytes: maxBytes,
|
|
163
|
+
maxFiles: cfg.artifacts.maxFiles,
|
|
164
|
+
maxDepth: cfg.artifacts.maxDepth
|
|
165
|
+
})) {
|
|
166
|
+
const key = relRoot.resolve(f.full);
|
|
167
|
+
if (seen.has(key)) continue;
|
|
168
|
+
seen.add(key);
|
|
169
|
+
const got = collectFile(f.full, { relLabel: () => rootLabel + '/' + f.rel }, modules, maxBytes, fileLines);
|
|
170
|
+
if (!got.length) skipped++; else scanned++;
|
|
171
|
+
for (const e of got) { e.abs = key; entries.push(e); }
|
|
172
|
+
}
|
|
173
|
+
// 存在性名录(供 TSR 这类引用型声明核验):只记名字不读内容
|
|
174
|
+
for (const f of iterate(root, {
|
|
175
|
+
exclude: cfg.artifacts.exclude,
|
|
176
|
+
matchInclude: () => true,
|
|
177
|
+
maxFiles: 20000,
|
|
178
|
+
maxDepth: cfg.artifacts.maxDepth
|
|
179
|
+
})) {
|
|
180
|
+
const key = 'p:' + relRoot.resolve(f.full);
|
|
181
|
+
if (seenPresence.has(key)) continue;
|
|
182
|
+
seenPresence.add(key);
|
|
183
|
+
presence.push(rootLabel + '/' + f.rel);
|
|
184
|
+
// 代码文件顺带数行数(「N 行」声明的证据),不抽取声明
|
|
185
|
+
if (CODE_EXT.has(extOf(f.rel)) && (!f.size || f.size <= CODE_LINE_MAX_BYTES)) {
|
|
186
|
+
try {
|
|
187
|
+
const text = fs.readFileSync(f.full, 'utf8');
|
|
188
|
+
fileLines.set(rootLabel + '/' + f.rel, countLines(text));
|
|
189
|
+
} catch (_) { /* 读失败就跳过,行数证据不是必须的 */ }
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
};
|
|
193
|
+
|
|
194
|
+
for (const rootInput of cfg.artifacts.roots) {
|
|
195
|
+
const abs = path.resolve(cfg.__baseDir || cfg.__cwd || process.cwd(), rootInput);
|
|
196
|
+
let st = null;
|
|
197
|
+
try { st = fs.statSync(abs); } catch (_) { continue; }
|
|
198
|
+
if (st.isDirectory()) { handleDir(abs); continue; }
|
|
199
|
+
if (!st.isFile()) continue;
|
|
200
|
+
const base = path.basename(abs);
|
|
201
|
+
const parent = path.basename(path.dirname(abs));
|
|
202
|
+
const rel = base;
|
|
203
|
+
const label = parent ? parent + '/' + base : base;
|
|
204
|
+
const key = relRoot.resolve(abs);
|
|
205
|
+
if (seen.has(key)) continue;
|
|
206
|
+
seen.add(key);
|
|
207
|
+
seenPresence.add('p:' + key);
|
|
208
|
+
presence.push(label);
|
|
209
|
+
const got = collectFile(abs, { relLabel: () => label }, modules, maxBytes, fileLines);
|
|
210
|
+
if (!got.length) skipped++; else scanned++;
|
|
211
|
+
for (const e of got) { e.abs = key; entries.push(e); }
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
// 存在性专用根:只读文件名,不读内容(TSR / TSQ / 哈希清单)
|
|
215
|
+
for (const rootInput of (cfg.artifacts.presenceRoots || [])) {
|
|
216
|
+
const abs = path.resolve(cfg.__baseDir || cfg.__cwd || process.cwd(), rootInput);
|
|
217
|
+
let st = null;
|
|
218
|
+
try { st = fs.statSync(abs); } catch (_) { continue; }
|
|
219
|
+
const rootLabel = st.isDirectory() ? (path.basename(abs) || '') : path.basename(path.dirname(abs));
|
|
220
|
+
const emit = (full, rel) => {
|
|
221
|
+
const key = 'p:' + path.resolve(full);
|
|
222
|
+
if (seenPresence.has(key)) return;
|
|
223
|
+
seenPresence.add(key);
|
|
224
|
+
presence.push(rootLabel ? rootLabel + '/' + rel : rel);
|
|
225
|
+
};
|
|
226
|
+
if (st.isFile()) { emit(abs, path.basename(abs)); continue; }
|
|
227
|
+
for (const f of iterate(abs, {
|
|
228
|
+
exclude: cfg.artifacts.exclude,
|
|
229
|
+
matchInclude: () => true,
|
|
230
|
+
maxFiles: 40000,
|
|
231
|
+
maxDepth: cfg.artifacts.maxDepth
|
|
232
|
+
})) emit(f.full, f.rel);
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
const denyRe = require('./walk').compilePatterns(cfg.artifacts.deny || []);
|
|
236
|
+
const denied = (e) => denyRe.some((re) => re.test(e.source + '::' + e.path) || re.test(e.source));
|
|
237
|
+
const keep = [];
|
|
238
|
+
for (const e of entries) { if (!denied(e)) keep.push(e); }
|
|
239
|
+
|
|
240
|
+
return { entries: keep, presence, scanned, skipped, denied: entries.length - keep.length, fileLines: Array.from(fileLines.entries()) };
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
// 过于通用的 token 不进倒排索引,否则候选集会爆炸("count" 之类几乎命中一切)。
|
|
244
|
+
const STOP_TOKENS = new Set([
|
|
245
|
+
'id', 'name', 'type', 'file', 'files', 'list', 'index', 'value', 'values', 'data', 'note', 'notes',
|
|
246
|
+
'title', 'version', 'status', 'state', 'path', 'source', 'key', 'keys', 'level', 'size', 'number',
|
|
247
|
+
'result', 'results', 'date', 'time', 'ts', 'msg', 'level0', 'text', 'item', 'items', 'ref', 'desc'
|
|
248
|
+
]);
|
|
249
|
+
|
|
250
|
+
/** primaryToken → 条目数组 + 全部 token → 条目数组。粒度放宽是为了 recall,质量由评分把关。 */
|
|
251
|
+
function buildIndex(artifacts) {
|
|
252
|
+
const byToken = new Map();
|
|
253
|
+
const push = (k, e) => {
|
|
254
|
+
if (!byToken.has(k)) byToken.set(k, []);
|
|
255
|
+
byToken.get(k).push(e);
|
|
256
|
+
};
|
|
257
|
+
for (const e of artifacts.entries) {
|
|
258
|
+
const withPrimary = new Set(e.keyTokens);
|
|
259
|
+
if (e.primaryToken) withPrimary.add(e.primaryToken);
|
|
260
|
+
for (const k of withPrimary) {
|
|
261
|
+
if (STOP_TOKENS.has(k)) continue;
|
|
262
|
+
if (k.length < 2 && !/^\d+$/.test(k)) continue;
|
|
263
|
+
push(k, e);
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
const presenceSquash = new Map();
|
|
267
|
+
for (const p of artifacts.presence) {
|
|
268
|
+
const sq = squash(p);
|
|
269
|
+
if (!presenceSquash.has(sq)) presenceSquash.set(sq, []);
|
|
270
|
+
presenceSquash.get(sq).push(p);
|
|
271
|
+
}
|
|
272
|
+
return { byToken, presence: artifacts.presence, presenceSquash, entries: artifacts.entries, fileLines: new Map(artifacts.fileLines || []) };
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
module.exports = { collectArtifacts, buildIndex, collectFile, extOf, moduleOf, TEXT_EXT, PRESENCE_EXT };
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
const fs = require('fs');
|
|
5
|
+
const path = require('path');
|
|
6
|
+
const crypto = require('crypto');
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* 增量缓存:以「文件集合的 mtime+size 指纹」为 key。
|
|
10
|
+
* 命中则完全跳过产出物解析(扫描阶段最贵的一段),这是性能的主要来源。
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
function ensureDir(dir) {
|
|
14
|
+
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
function fileSignature(roots, cfg) {
|
|
18
|
+
const list = [];
|
|
19
|
+
const walkDir = (dir, rootAbs, depth) => {
|
|
20
|
+
let ents;
|
|
21
|
+
try { ents = fs.readdirSync(dir, { withFileTypes: true }); } catch (_) { return; }
|
|
22
|
+
for (const e of ents) {
|
|
23
|
+
const full = path.join(dir, e.name);
|
|
24
|
+
const rel = path.relative(rootAbs, full).split(path.sep).join('/');
|
|
25
|
+
if (e.isDirectory()) {
|
|
26
|
+
if (cfg.exclude.some((p) => new RegExp('^' + p.replace(/\*\*/g, '.*').replace(/\*/g, '[^/]*') + '/?$').test(rel + '/'))) continue;
|
|
27
|
+
if (cfg.maxDepth && rel.split('/').length > cfg.maxDepth) continue;
|
|
28
|
+
walkDir(full, rootAbs, depth + 1);
|
|
29
|
+
continue;
|
|
30
|
+
}
|
|
31
|
+
let st = null;
|
|
32
|
+
try { st = fs.statSync(full); } catch (_) { continue; }
|
|
33
|
+
list.push([rel, st.mtimeMs, st.size]);
|
|
34
|
+
}
|
|
35
|
+
};
|
|
36
|
+
for (const r of roots) {
|
|
37
|
+
const abs = path.resolve(r);
|
|
38
|
+
let st = null;
|
|
39
|
+
try { st = fs.statSync(abs); } catch (_) { continue; }
|
|
40
|
+
if (st.isDirectory()) walkDir(abs, abs, 0);
|
|
41
|
+
else list.push([path.basename(abs), st.mtimeMs, st.size]);
|
|
42
|
+
}
|
|
43
|
+
list.sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0));
|
|
44
|
+
return crypto.createHash('sha256').update(JSON.stringify(list)).digest('hex').slice(0, 32);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function cachePaths(cfg, cwd) {
|
|
48
|
+
const dir = path.resolve(cwd, cfg.cache.dir || '.verifact');
|
|
49
|
+
return { dir, file: path.join(dir, 'artifact-cache.json'), history: path.join(dir, 'history.jsonl') };
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function readArtifactCache(cfg, cwd, roots) {
|
|
53
|
+
if (!cfg.cache.enabled) return { hit: false, reason: 'disabled' };
|
|
54
|
+
const p = cachePaths(cfg, cwd);
|
|
55
|
+
if (!fs.existsSync(p.file)) return { hit: false, reason: 'no_cache_file' };
|
|
56
|
+
try {
|
|
57
|
+
const raw = JSON.parse(fs.readFileSync(p.file, 'utf8'));
|
|
58
|
+
const sig = fileSignature(roots, { exclude: cfg.artifacts.exclude, maxDepth: cfg.artifacts.maxDepth });
|
|
59
|
+
if (raw.signature !== sig) return { hit: false, reason: 'stale', staged: raw };
|
|
60
|
+
return { hit: true, entries: raw.entries || [], presence: raw.presence || [], scanned: raw.scanned || 0, skipped: raw.skipped || 0, fileLines: raw.fileLines || [], signature: sig };
|
|
61
|
+
} catch (_) {
|
|
62
|
+
return { hit: false, reason: 'corrupt' };
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function writeArtifactCache(cfg, cwd, roots, artifacts) {
|
|
67
|
+
if (!cfg.cache.enabled) return false;
|
|
68
|
+
const p = cachePaths(cfg, cwd);
|
|
69
|
+
try {
|
|
70
|
+
ensureDir(p.dir);
|
|
71
|
+
const payload = {
|
|
72
|
+
version: 1,
|
|
73
|
+
signature: fileSignature(roots, { exclude: cfg.artifacts.exclude, maxDepth: cfg.artifacts.maxDepth }),
|
|
74
|
+
entries: artifacts.entries,
|
|
75
|
+
presence: artifacts.presence,
|
|
76
|
+
scanned: artifacts.scanned,
|
|
77
|
+
skipped: artifacts.skipped,
|
|
78
|
+
fileLines: artifacts.fileLines || []
|
|
79
|
+
};
|
|
80
|
+
fs.writeFileSync(p.file, JSON.stringify(payload));
|
|
81
|
+
return true;
|
|
82
|
+
} catch (_) {
|
|
83
|
+
return false;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
module.exports = { readArtifactCache, writeArtifactCache, fileSignature, cachePaths, ensureDir };
|