zipcalm 0.0.0-stage → 1.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +260 -2
  3. package/bin/zipcalm.js +191 -0
  4. package/package.json +67 -4
  5. package/server/app.js +309 -0
  6. package/server/boot.js +145 -0
  7. package/server/config/defaults.js +153 -0
  8. package/server/config/index.js +140 -0
  9. package/server/config/schema.js +244 -0
  10. package/server/core/auth/password.js +89 -0
  11. package/server/core/auth/store.js +261 -0
  12. package/server/core/compress/blacklist.js +236 -0
  13. package/server/core/compress/capRegistry.js +171 -0
  14. package/server/core/compress/deleter.js +285 -0
  15. package/server/core/compress/hardlinkScan.js +67 -0
  16. package/server/core/compress/orchestrator.js +783 -0
  17. package/server/core/compress/preflight.js +376 -0
  18. package/server/core/compress/registry.js +240 -0
  19. package/server/core/compress/sourceWalk.js +84 -0
  20. package/server/core/compress/verify.js +47 -0
  21. package/server/core/engineRunner.js +369 -0
  22. package/server/core/fs/list.js +164 -0
  23. package/server/core/fs/platformProfile.js +167 -0
  24. package/server/core/fs/repairNames.js +150 -0
  25. package/server/core/fs/roots.js +178 -0
  26. package/server/core/fs/safePath.js +173 -0
  27. package/server/core/fs/search.js +108 -0
  28. package/server/core/fs/stat.js +110 -0
  29. package/server/core/fs/systemTrash.js +212 -0
  30. package/server/core/fs/writeOps.js +257 -0
  31. package/server/core/hardlink/executor.js +106 -0
  32. package/server/core/hardlink/scanner.js +193 -0
  33. package/server/core/metrics/collector.js +279 -0
  34. package/server/core/metrics/procfs.js +107 -0
  35. package/server/core/task/events.js +82 -0
  36. package/server/core/task/manager.js +987 -0
  37. package/server/core/task/recovery.js +116 -0
  38. package/server/core/task/store.js +240 -0
  39. package/server/http/middleware/auth.js +130 -0
  40. package/server/http/middleware/error.js +54 -0
  41. package/server/http/middleware/logger.js +27 -0
  42. package/server/http/middleware/pathGuard.js +119 -0
  43. package/server/http/router.js +29 -0
  44. package/server/http/routes/auth.js +115 -0
  45. package/server/http/routes/blacklist.js +97 -0
  46. package/server/http/routes/capabilities.js +21 -0
  47. package/server/http/routes/compress.js +339 -0
  48. package/server/http/routes/fs.js +109 -0
  49. package/server/http/routes/hardlink.js +160 -0
  50. package/server/http/routes/health.js +52 -0
  51. package/server/http/routes/metrics.js +36 -0
  52. package/server/http/routes/tasks.js +304 -0
  53. package/server/lib/atomic-json.js +118 -0
  54. package/server/lib/errors.js +145 -0
  55. package/server/lib/fs-probe.js +25 -0
  56. package/server/lib/logger.js +68 -0
  57. package/server/lib/runtime-guard.js +49 -0
  58. package/server/lib/throttle.js +42 -0
  59. package/server/plugins/capabilities/password/index.js +300 -0
  60. package/server/plugins/capabilities/password/methods/aes256.js +31 -0
  61. package/server/plugins/capabilities/password/methods/zip20.js +29 -0
  62. package/server/plugins/formats/_shared/createGuards.js +26 -0
  63. package/server/plugins/formats/_shared/entrySafety.js +174 -0
  64. package/server/plugins/formats/_shared/nameRepair.js +123 -0
  65. package/server/plugins/formats/_shared/streams.js +18 -0
  66. package/server/plugins/formats/_shared/tar/compression.js +114 -0
  67. package/server/plugins/formats/_shared/tar/reader.js +74 -0
  68. package/server/plugins/formats/_shared/tar/writer.js +206 -0
  69. package/server/plugins/formats/_shared/tarFamily.js +207 -0
  70. package/server/plugins/formats/rar/index.js +354 -0
  71. package/server/plugins/formats/rar/js-engine.js +122 -0
  72. package/server/plugins/formats/rar/js-runner.mjs +129 -0
  73. package/server/plugins/formats/rar/unrar.js +266 -0
  74. package/server/plugins/formats/sevenzip/engine.js +355 -0
  75. package/server/plugins/formats/sevenzip/index.js +461 -0
  76. package/server/plugins/formats/sevenzip/wasm-runner.mjs +72 -0
  77. package/server/plugins/formats/tar/index.js +19 -0
  78. package/server/plugins/formats/targz/index.js +19 -0
  79. package/server/plugins/formats/tarzst/index.js +26 -0
  80. package/server/plugins/formats/zip/index.js +587 -0
  81. package/shared/app-version.js +7 -0
  82. package/shared/capabilities.js +21 -0
  83. package/shared/compress-modes.js +38 -0
  84. package/shared/error-codes.js +117 -0
  85. package/shared/format-list.js +101 -0
  86. package/shared/hardlink-hint.js +21 -0
  87. package/shared/task-phase.js +62 -0
  88. package/shared/task-status.js +84 -0
  89. package/web/dist/assets/index-D3zjq7N2.js +131 -0
  90. package/web/dist/assets/index-D9Vdmrb9.css +1 -0
  91. package/web/dist/index.html +18 -0
@@ -0,0 +1,261 @@
1
+ // 认证存储(M1):Session + Bearer 双通道
2
+ //
3
+ // 契约(docs/M0/API规范.md §二):
4
+ // - Session:HttpOnly Cookie `zc_session`,7 天**滑动续期**;
5
+ // - Bearer:`Authorization: Bearer <token>`,30 天;
6
+ // - 两者同时存在时 **Bearer 优先**;
7
+ // - 凭据只存哈希(sha256),磁盘上不出现明文(INV-16)。
8
+ //
9
+ // 会话放内存(重启即失效,属可接受语义);Bearer token 落盘持久化,
10
+ // 这样 APK/脚本重启服务后仍可用(数据模型 §六 storage/auth-tokens.json)。
11
+
12
+ import path from 'node:path';
13
+
14
+ import { readJson, writeJsonAtomic } from '../../lib/atomic-json.js';
15
+ import { randomToken, sha256 } from './password.js';
16
+
17
+ /** 滑动续期阈值:剩余时间低于 TTL 的一半时才重新签发 Cookie(避免每个请求都 Set-Cookie) */
18
+ const RENEW_THRESHOLD_RATIO = 0.5;
19
+ /** token 落盘节流窗口:把同一窗口内的多次变更合并成一次原子写(登录/续期/清理都会触发变更) */
20
+ const TOKEN_PERSIST_THROTTLE_MS = 50;
21
+ /** 过期条目清理周期(1 分钟;定时器 unref,不阻止进程退出) */
22
+ const CLEANUP_INTERVAL_MS = 60 * 1000;
23
+
24
+ /**
25
+ * 清掉已过期的条目并返回清掉的条数。
26
+ * token 与 session 两张表结构相同(都是 {expiresAt}),所以共用这一份实现。
27
+ */
28
+ function pruneExpired(map, nowTs) {
29
+ let removed = 0;
30
+ for (const [key, value] of map) {
31
+ if (value.expiresAt > nowTs) continue;
32
+ map.delete(key);
33
+ removed += 1;
34
+ }
35
+ return removed;
36
+ }
37
+
38
+ /** 登录失败计数按限流窗口过期(窗口外的不再计入) */
39
+ function pruneLoginFailures(records, windowMs, nowTs) {
40
+ for (const [ip, record] of records) {
41
+ if (nowTs - record.firstAt > windowMs) records.delete(ip);
42
+ }
43
+ }
44
+
45
+ export function createAuthStore({ config, logger, dataDir }) {
46
+ const sessionTtlMs = config.sessionTtlMs;
47
+ const tokenTtlMs = config.tokenTtlMs;
48
+ const username = config.username;
49
+ const tokenFilePath = path.join(dataDir, 'auth-tokens.json');
50
+
51
+ /** Map<sha256(sid), {user, expiresAt, issuedAt, lastSeenAt}> */
52
+ const sessions = new Map();
53
+ /** Map<sha256(token), {user, expiresAt, issuedAt}> */
54
+ const tokens = new Map();
55
+ /** 登录失败计数:Map<ip, {count, firstAt}> */
56
+ const loginFailures = new Map();
57
+
58
+ let persistTimer = null;
59
+
60
+ // ===== 持久化(只存哈希 + 过期时间)=====
61
+
62
+ /** 一条落盘 token 是否还能用(字段合法 + 未过期);不合法/过期一律丢弃 */
63
+ function isUsableToken(tokenEntry, nowTs) {
64
+ if (!tokenEntry || typeof tokenEntry.hash !== 'string' || typeof tokenEntry.expiresAt !== 'number') return false;
65
+ return tokenEntry.expiresAt > nowTs;
66
+ }
67
+
68
+ /** 把落盘记录灌进内存索引(过期与不合法条目直接跳过) */
69
+ function restoreTokens(entries, nowTs) {
70
+ for (const tokenEntry of entries) {
71
+ if (!isUsableToken(tokenEntry, nowTs)) continue;
72
+ tokens.set(tokenEntry.hash, {
73
+ user: tokenEntry.user || username,
74
+ expiresAt: tokenEntry.expiresAt,
75
+ issuedAt: tokenEntry.issuedAt || nowTs,
76
+ });
77
+ }
78
+ }
79
+
80
+ async function load() {
81
+ const storedTokens = await readJson(tokenFilePath, { tokens: [] });
82
+ if (!storedTokens.ok) {
83
+ if (storedTokens.reason !== 'not-found') {
84
+ logger.warn('auth token store unreadable, starting empty', { reason: storedTokens.reason });
85
+ }
86
+ return;
87
+ }
88
+ const entries = Array.isArray(storedTokens.data && storedTokens.data.tokens) ? storedTokens.data.tokens : [];
89
+ restoreTokens(entries, Date.now());
90
+ if (tokens.size > 0) logger.info('auth tokens restored', { count: tokens.size });
91
+ }
92
+
93
+ function schedulePersist() {
94
+ if (persistTimer) return;
95
+ persistTimer = setTimeout(() => {
96
+ persistTimer = null;
97
+ persist().catch((error) => logger.error('persist auth tokens failed', { error: error.message }));
98
+ }, TOKEN_PERSIST_THROTTLE_MS);
99
+ if (typeof persistTimer.unref === 'function') persistTimer.unref();
100
+ }
101
+
102
+ async function persist() {
103
+ const payload = {
104
+ version: 1,
105
+ updatedAt: new Date().toISOString(),
106
+ tokens: [...tokens.entries()].map(([hash, value]) => ({
107
+ hash, // sha256(token):磁盘上无法反推出可登录明文
108
+ user: value.user,
109
+ issuedAt: value.issuedAt,
110
+ expiresAt: value.expiresAt,
111
+ })),
112
+ };
113
+ await writeJsonAtomic(tokenFilePath, payload);
114
+ }
115
+
116
+ /**
117
+ * 立即落盘(关停时调用)。
118
+ * 为什么需要:签发 token 后走的是 50ms 节流;若进程在这 50ms 内退出,
119
+ * 刚发出的 token 就没落盘 —— 重启后拿着有效凭据的客户端会突然 401(实测踩过)。
120
+ */
121
+ async function flush() {
122
+ if (persistTimer) {
123
+ clearTimeout(persistTimer);
124
+ persistTimer = null;
125
+ }
126
+ await persist();
127
+ }
128
+
129
+ // ===== Session =====
130
+
131
+ function createSession(user = username) {
132
+ const sid = randomToken(32);
133
+ const now = Date.now();
134
+ const expiresAt = now + sessionTtlMs;
135
+ sessions.set(sha256(sid), { user, expiresAt, issuedAt: now, lastSeenAt: now });
136
+ return { sid, expiresAt };
137
+ }
138
+
139
+ /**
140
+ * 校验 Session;命中且已过续期阈值时返回 renewed=true(调用方据此重发 Cookie)。
141
+ */
142
+ function verifySession(sid) {
143
+ if (!sid) return null;
144
+ const key = sha256(sid);
145
+ const session = sessions.get(key);
146
+ if (!session) return null;
147
+
148
+ const now = Date.now();
149
+ if (session.expiresAt <= now) {
150
+ sessions.delete(key);
151
+ return null;
152
+ }
153
+
154
+ session.lastSeenAt = now;
155
+ const remaining = session.expiresAt - now;
156
+ let renewed = false;
157
+ if (remaining < sessionTtlMs * RENEW_THRESHOLD_RATIO) {
158
+ session.expiresAt = now + sessionTtlMs;
159
+ renewed = true;
160
+ }
161
+ return { key, user: session.user, expiresAt: session.expiresAt, renewed };
162
+ }
163
+
164
+ function revokeSession(sid) {
165
+ if (!sid) return false;
166
+ return sessions.delete(sha256(sid));
167
+ }
168
+
169
+ // ===== Bearer Token =====
170
+
171
+ function createToken(user = username) {
172
+ const token = randomToken(32);
173
+ const now = Date.now();
174
+ const expiresAt = now + tokenTtlMs;
175
+ tokens.set(sha256(token), { user, expiresAt, issuedAt: now });
176
+ schedulePersist();
177
+ return { token, expiresAt };
178
+ }
179
+
180
+ function verifyToken(token) {
181
+ if (!token) return null;
182
+ const key = sha256(token);
183
+ const record = tokens.get(key);
184
+ if (!record) return null;
185
+ if (record.expiresAt <= Date.now()) {
186
+ tokens.delete(key);
187
+ schedulePersist();
188
+ return null;
189
+ }
190
+ return { key, user: record.user, expiresAt: record.expiresAt };
191
+ }
192
+
193
+ function revokeToken(token) {
194
+ const key = sha256(token);
195
+ const existed = tokens.delete(key);
196
+ if (existed) schedulePersist();
197
+ return existed;
198
+ }
199
+
200
+ // ===== 登录失败限流(内存计数,够 M1 用)=====
201
+
202
+ function isLoginBlocked(ip, limit) {
203
+ const record = loginFailures.get(ip);
204
+ if (!record) return false;
205
+ if (Date.now() - record.firstAt > limit.windowMs) {
206
+ loginFailures.delete(ip);
207
+ return false;
208
+ }
209
+ return record.count >= limit.max;
210
+ }
211
+
212
+ function recordLoginFailure(ip, limit) {
213
+ const now = Date.now();
214
+ const record = loginFailures.get(ip);
215
+ if (!record || now - record.firstAt > limit.windowMs) {
216
+ loginFailures.set(ip, { count: 1, firstAt: now });
217
+ return;
218
+ }
219
+ record.count += 1;
220
+ }
221
+
222
+ function clearLoginFailures(ip) {
223
+ loginFailures.delete(ip);
224
+ }
225
+
226
+ // ===== 过期清理(定时器 unref,不阻止进程退出)=====
227
+
228
+ function startCleanup() {
229
+ const timer = setInterval(() => {
230
+ const now = Date.now();
231
+ const removedTokens = pruneExpired(tokens, now);
232
+ pruneExpired(sessions, now);
233
+ pruneLoginFailures(loginFailures, config.loginRateLimit.windowMs, now);
234
+ if (removedTokens > 0) {
235
+ schedulePersist();
236
+ logger.debug('expired auth tokens cleaned', { removedTokens });
237
+ }
238
+ }, CLEANUP_INTERVAL_MS);
239
+ if (typeof timer.unref === 'function') timer.unref();
240
+ return () => clearInterval(timer);
241
+ }
242
+
243
+ return {
244
+ load,
245
+ flush,
246
+ sessionTtlMs,
247
+ tokenTtlMs,
248
+ createSession,
249
+ verifySession,
250
+ revokeSession,
251
+ createToken,
252
+ verifyToken,
253
+ revokeToken,
254
+ isLoginBlocked,
255
+ recordLoginFailure,
256
+ clearLoginFailures,
257
+ startCleanup,
258
+ /** 仅用于测试/诊断,不暴露明文 */
259
+ stats: () => ({ sessions: sessions.size, tokens: tokens.size }),
260
+ };
261
+ }
@@ -0,0 +1,236 @@
1
+ // 压缩黑名单(三层:全局默认 / 用户自定义 / 任务级临时)+ 匹配与「命中哪一层」
2
+ //
3
+ // 契约:
4
+ // - docs/04 §三:三层表 + 「命中后跳过但保留在原目录」(绝不能进删除集合)
5
+ // - docs/M0/API规范.md §4.5:三端点形状;§4.2 目录列表的 blacklisted/blacklistLayer;
6
+ // §compress/preflight 的 skipped[]{path,pattern,layer}
7
+ // - docs/09 §五:黑名单匹配归服务端(不信任前端)
8
+ //
9
+ // 为什么要单独一个模块:匹配规则会被三处消费 —— 预检(告知"会跳过几个")、真实打包(真的不压)、
10
+ // 目录列表(给每个条目标注)。三处必须用**同一份判定**,否则"预检说跳过 3 个、实际压了 3 个"这种
11
+ // 漂移几乎必然会来。所以这里既提供纯函数(compile/resolve/match),也提供用户层的读写。
12
+
13
+ import fsp from 'node:fs/promises';
14
+ import path from 'node:path';
15
+
16
+ import { writeJsonAtomic } from '../../lib/atomic-json.js';
17
+
18
+ /** 层名(对外字段 blacklistLayer 的取值;顺序即优先级:越靠前越具体) */
19
+ export const BLACKLIST_LAYERS = Object.freeze({
20
+ TASK: 'task',
21
+ USER: 'user',
22
+ GLOBAL: 'global',
23
+ });
24
+
25
+ /**
26
+ * 全局默认层:已压缩格式 + 系统垃圾(不可删、不可改;可被任务级开关整体关掉)。
27
+ * 依据 docs/04 §四「智能跳过」表:jpg/png/mp4/mp3、zip/7z/rar/gz、docx/xlsx/pptx 再压几乎无收益。
28
+ * 系统垃圾(Thumbs.db/.DS_Store/@eaDir)本来就不该进包。
29
+ */
30
+ export const GLOBAL_PATTERNS = Object.freeze([
31
+ // 只放**系统垃圾**:这些文件进包没有任何意义(缩略图缓存、Finder 元数据、群晖元目录)。
32
+ //
33
+ // 曾经默认还包含 jpg/png/mp4/zip/docx 等 12 个"已压缩格式",已移除 ——
34
+ // 原因:打包不总是为了省空间("把照片打包发给别人"是常见需求),默认静默跳过会让打包结果
35
+ // 与预期不符。机制保留:这些规则可以显式加进用户层,或在任务级用 extraPatterns 临时加。
36
+ 'Thumbs.db',
37
+ '.DS_Store',
38
+ '@eaDir',
39
+ ]);
40
+
41
+ /** 契约 §4.5 的校验上限 */
42
+ export const MAX_PATTERN_LENGTH = 128;
43
+ export const MAX_PATTERNS = 200;
44
+
45
+ /** 用户层文件名(独立文件:config.json 是启动期只读快照,没有写回通道,不适合放用户可改的设置) */
46
+ const USER_FILE = 'blacklist-user.json';
47
+
48
+ /** 用户层默认值 */
49
+ function emptyUserLayer() {
50
+ return { enabled: true, patterns: [], updatedAt: null };
51
+ }
52
+
53
+ /**
54
+ * 校验一串 pattern(PUT /blacklist/user 用)。
55
+ * 契约要求:单条 ≤128 字符、总数 ≤200、拒绝绝对路径与 `/`。
56
+ * 另外这里把"空串/纯空白"也拒掉(否则会变成"什么都不匹配"的死条目,用户看不出为什么没生效)。
57
+ * @returns {{ok: true, patterns: string[]} | {ok: false, message: string}}
58
+ */
59
+ /** 单条校验(返回规范化后的字符串,或错误消息) */
60
+ function validateOnePattern(raw) {
61
+ if (typeof raw !== 'string') return { ok: false, message: 'patterns 里每一项都必须是字符串' };
62
+ const pattern = raw.trim();
63
+ if (pattern.length === 0) return { ok: false, message: 'patterns 里不允许空字符串' };
64
+ if (pattern.length > MAX_PATTERN_LENGTH) return { ok: false, message: `单条 pattern 最多 ${MAX_PATTERN_LENGTH} 字符` };
65
+ // 绝对路径与分隔符:黑名单只描述"名字/扩展名",不是路径规则(否则会误以为支持目录前缀)
66
+ if (pattern.includes('/') || pattern.includes('\\')) return { ok: false, message: `pattern 不允许包含路径分隔符:${pattern}` };
67
+ if (path.isAbsolute(pattern)) return { ok: false, message: `pattern 不允许是绝对路径:${pattern}` };
68
+ return { ok: true, pattern };
69
+ }
70
+
71
+ export function validatePatterns(rawPatterns) {
72
+ if (!Array.isArray(rawPatterns)) return { ok: false, message: 'patterns 必须是字符串数组' };
73
+ if (rawPatterns.length > MAX_PATTERNS) return { ok: false, message: `patterns 最多 ${MAX_PATTERNS} 条` };
74
+
75
+ const patterns = [];
76
+ for (const raw of rawPatterns) {
77
+ const checked = validateOnePattern(raw);
78
+ if (!checked.ok) return checked;
79
+ if (!patterns.includes(checked.pattern)) patterns.push(checked.pattern);
80
+ }
81
+ return { ok: true, patterns };
82
+ }
83
+
84
+ /**
85
+ * 单条 pattern → 判定器。
86
+ * 支持的三种形态(契约 §4.5「按扩展名 / 按文件名」+ 全局层里的 @eaDir 目录名):
87
+ * - `*.jpg` → 扩展名(大小写不敏感;`*.gz` 也能命中 `a.tar.gz`,因为按后缀比)
88
+ * - `Thumbs.db` / `.DS_Store` → 精确文件名(大小写不敏感:NAS 上大小写混写很常见)
89
+ * - `@eaDir` → 任一路径段同名(这样 @eaDir 里的文件也一并跳过)
90
+ * @param {string} pattern
91
+ * @returns {{pattern: string, test: (name: string, segments: string[]) => boolean}}
92
+ */
93
+ export function compilePattern(pattern) {
94
+ const lowered = pattern.toLowerCase();
95
+ if (lowered.startsWith('*.')) {
96
+ const suffix = lowered.slice(1); // '*.jpg' → '.jpg'
97
+ return {
98
+ pattern,
99
+ test: (name) => name.toLowerCase().endsWith(suffix),
100
+ };
101
+ }
102
+ return {
103
+ pattern,
104
+ test: (name, segments) => name.toLowerCase() === lowered
105
+ || (Array.isArray(segments) && segments.some((segment) => segment.toLowerCase() === lowered)),
106
+ };
107
+ }
108
+
109
+ /**
110
+ * 把三层解析成"按优先级排好的判定层"(task > user > global)。
111
+ * 关掉的层不参与匹配;`enabled:false` 的任务级开关表示**整条黑名单不生效**(三层都跳过)。
112
+ * @param {object} input
113
+ * @param {{enabled?: boolean, patterns?: string[]}} [input.task] 任务级临时层(extraPatterns)
114
+ * @param {{enabled?: boolean, patterns?: string[]}} [input.user] 用户层
115
+ * @param {{enabled?: boolean}} [input.global] 全局层(默认开)
116
+ * @returns {Array<{layer: string, enabled: boolean, patterns: string[]}>}
117
+ */
118
+ export function resolveLayers({ task, user, global } = {}) {
119
+ const layers = [];
120
+ if (task && task.enabled !== false && Array.isArray(task.patterns) && task.patterns.length > 0) {
121
+ layers.push({ layer: BLACKLIST_LAYERS.TASK, patterns: task.patterns });
122
+ }
123
+ if (user && user.enabled !== false && Array.isArray(user.patterns) && user.patterns.length > 0) {
124
+ layers.push({ layer: BLACKLIST_LAYERS.USER, patterns: user.patterns });
125
+ }
126
+ if (!global || global.enabled !== false) {
127
+ layers.push({ layer: BLACKLIST_LAYERS.GLOBAL, patterns: [...GLOBAL_PATTERNS] });
128
+ }
129
+ return layers;
130
+ }
131
+
132
+ /**
133
+ * 判定一个条目。
134
+ * @param {Array<{layer: string, patterns: string[]}>} layers resolveLayers() 的结果(已按优先级排列)
135
+ * @param {{name: string, segments?: string[]}} entry name=文件名;segments=相对路径的各段(用于目录名规则)
136
+ * @returns {{pattern: string, layer: string} | null} 命中返回到层与 pattern,未命中 null
137
+ */
138
+ export function matchEntry(layers, entry) {
139
+ if (!Array.isArray(layers) || layers.length === 0) return null;
140
+ const name = String((entry && entry.name) || '');
141
+ if (!name) return null;
142
+ const segments = Array.isArray(entry.segments) ? entry.segments : [];
143
+ for (const layer of layers) {
144
+ for (const pattern of layer.patterns) {
145
+ if (compilePattern(pattern).test(name, segments)) {
146
+ return { pattern, layer: layer.layer };
147
+ }
148
+ }
149
+ }
150
+ return null;
151
+ }
152
+
153
+ /** 有效 pattern 条数(GET /blacklist 的 effectiveCount:三层去重后总数) */
154
+ export function countEffective(layers) {
155
+ const seen = new Set();
156
+ for (const layer of layers || []) {
157
+ for (const pattern of layer.patterns || []) seen.add(pattern.toLowerCase());
158
+ }
159
+ return seen.size;
160
+ }
161
+
162
+ /**
163
+ * 黑名单服务:持有用户层的读写(其余两层由调用方传入)。
164
+ * 用户层落盘到 `<dataDir>/blacklist-user.json`,原子写(写坏一个文件不该让设置彻底失效)。
165
+ */
166
+ export function createBlacklistService({ dataDir, logger }) {
167
+ const userFile = path.join(dataDir, USER_FILE);
168
+ let cached = null; // 懒加载缓存:GET /blacklist 与预检都可能频繁读
169
+
170
+ /** 读用户层(缓存 + 文件;读取失败按默认值,并告警 —— 不能因为一个坏文件让压缩任务起不来) */
171
+ async function loadUser() {
172
+ if (cached) return cached;
173
+ try {
174
+ const text = await fsp.readFile(userFile, 'utf8');
175
+ const parsed = JSON.parse(text);
176
+ const checked = validatePatterns(parsed && parsed.patterns);
177
+ cached = {
178
+ enabled: parsed && parsed.enabled !== false,
179
+ patterns: checked.ok ? checked.patterns : [],
180
+ updatedAt: Number.isFinite(parsed && parsed.updatedAt) ? parsed.updatedAt : null,
181
+ };
182
+ if (!checked.ok) logger.warn('blacklist 用户层文件不合法,已按空处理', { reason: checked.message });
183
+ } catch (error) {
184
+ if (error.code !== 'ENOENT') logger.warn('blacklist 用户层读取失败,按默认值处理', { error: error.message });
185
+ cached = emptyUserLayer();
186
+ }
187
+ return cached;
188
+ }
189
+
190
+ /** 全量替换用户层(幂等);返回替换后的用户层 */
191
+ async function replaceUser({ patterns, enabled }) {
192
+ const checked = validatePatterns(patterns);
193
+ if (!checked.ok) {
194
+ const error = new Error(checked.message);
195
+ error.validation = true;
196
+ throw error;
197
+ }
198
+ const next = {
199
+ enabled: enabled !== false,
200
+ patterns: checked.patterns,
201
+ updatedAt: Date.now(),
202
+ };
203
+ await fsp.mkdir(dataDir, { recursive: true });
204
+ await writeJsonAtomic(userFile, { version: 1, ...next });
205
+ cached = next;
206
+ return next;
207
+ }
208
+
209
+ /** 三层快照(GET /blacklist 的 data;global 的 enabled 恒 true —— 契约 locked 表示不可改) */
210
+ async function snapshot({ taskPatterns = [] } = {}) {
211
+ const user = await loadUser();
212
+ const layers = resolveLayers({
213
+ task: { enabled: true, patterns: taskPatterns },
214
+ user,
215
+ global: { enabled: true },
216
+ });
217
+ return {
218
+ global: { locked: true, enabled: true, patterns: [...GLOBAL_PATTERNS] },
219
+ user: { enabled: user.enabled, patterns: [...user.patterns], updatedAt: user.updatedAt },
220
+ effectiveCount: countEffective(layers),
221
+ };
222
+ }
223
+
224
+ /** 解析出本次判定用的层(taskPatterns = 任务级临时层;enabled=false 表示整条黑名单关掉) */
225
+ async function layersFor({ enabled = true, taskPatterns = [] } = {}) {
226
+ if (!enabled) return [];
227
+ const user = await loadUser();
228
+ return resolveLayers({
229
+ task: { enabled: true, patterns: taskPatterns },
230
+ user,
231
+ global: { enabled: true },
232
+ });
233
+ }
234
+
235
+ return { loadUser, replaceUser, snapshot, layersFor, userFile };
236
+ }
@@ -0,0 +1,171 @@
1
+ // 能力注册表(docs/19 §三/§五 图 1:扫 capabilities/<id>/index.js)
2
+ //
3
+ // 照 registry.js 的模式:
4
+ // - 自动发现:扫描 server/plugins/capabilities/<id>/index.js;
5
+ // - 启动校验:必需成员/方法齐全;**不合规只禁用该包并告警,不影响其他包**;
6
+ // - 额外对账:appliesTo 声明的格式 id 必须存在于格式注册表(双向声明任一方缺失即不生效,
7
+ // docs/19 §4.2——避免「格式说支持但能力没实现」或反之);
8
+ // - 运行时重扫:目录 mtime 变化即重扫(与 registry 一致,加能力包零改动生效)。
9
+ //
10
+ // 与 registry 的差异:能力不参与「按扩展名匹配」,索引只有 id;
11
+ // 查询口径是「某格式启用的能力集合」= 格式 capabilities ∩ 能力 appliesTo ∩ 能力已加载。
12
+
13
+ import fs from 'node:fs/promises';
14
+ import path from 'node:path';
15
+ import { fileURLToPath } from 'node:url';
16
+
17
+ import { CAPABILITY_IDS } from '../../../shared/capabilities.js';
18
+
19
+ const DEFAULT_CAPS_DIR = path.resolve(
20
+ path.dirname(fileURLToPath(import.meta.url)),
21
+ '../../plugins/capabilities',
22
+ );
23
+
24
+ /** 能力契约必需成员(docs/19 §4.2) */
25
+ const REQUIRED_MEMBERS = ['id', 'name', 'appliesTo', 'params'];
26
+ const REQUIRED_METHODS = ['validateParams', 'prepareCreate', 'verify', 'prepareExtract', 'redact'];
27
+
28
+ function validateCapability(cap, source) {
29
+ const problems = [];
30
+ for (const member of REQUIRED_MEMBERS) {
31
+ if (cap[member] === undefined) problems.push(`缺少成员 ${member}`);
32
+ }
33
+ for (const method of REQUIRED_METHODS) {
34
+ if (typeof cap[method] !== 'function') problems.push(`缺少方法 ${method}()`);
35
+ }
36
+ if (Array.isArray(cap.appliesTo) === false) problems.push('appliesTo 必须是数组');
37
+ if (problems.length > 0) {
38
+ return { ok: false, reason: `${source}: ${problems.join(';')}` };
39
+ }
40
+ return { ok: true };
41
+ }
42
+
43
+ export function createCapabilityRegistry(options = {}) {
44
+ const logger = options.logger || { info() {}, warn() {} };
45
+ const capsDir = options.capsDir || DEFAULT_CAPS_DIR;
46
+ /** 格式 id 列表(对账用;由 orchestrator/启动流程喂进来或用 isFormatKnown 动态查) */
47
+ const knownFormatIds = new Set(options.formatIds || []);
48
+ const byId = new Map();
49
+ const loadFailures = [];
50
+ let lastScanMtimeMs = 0;
51
+
52
+ /** 已加载能力 ∩ 已声明该能力的格式 ∩ 能力 appliesTo → 三方交集查询 */
53
+ function capabilitiesForFormat(formatId) {
54
+ const result = [];
55
+ for (const cap of byId.values()) {
56
+ if (Array.isArray(cap.appliesTo) && cap.appliesTo.includes(formatId)) result.push(cap);
57
+ }
58
+ return result;
59
+ }
60
+
61
+ /**
62
+ * 校验「格式声明了能力但能力不存在/不适用」——PW-4 的数据面:后端独立判定。
63
+ * 返回 {ok, reason?};格式未声明任何能力 → ok(能力交集为空,装配循环自然跳过)。
64
+ */
65
+ function checkFormatCapabilities(plugin) {
66
+ const declared = plugin.capabilities || [];
67
+ for (const capId of declared) {
68
+ const cap = byId.get(capId);
69
+ if (!cap) return { ok: false, reason: `格式 ${plugin.id} 声明的能力 ${capId} 未加载` };
70
+ if (!cap.appliesTo.includes(plugin.id)) {
71
+ return { ok: false, reason: `能力 ${capId} 不适用于格式 ${plugin.id}` };
72
+ }
73
+ }
74
+ return { ok: true };
75
+ }
76
+
77
+ async function rescan() {
78
+ let dirStat;
79
+ try {
80
+ dirStat = await fs.stat(capsDir);
81
+ } catch {
82
+ return; // 目录不存在 = 尚无任何能力插件(合法状态)
83
+ }
84
+ if (dirStat.mtimeMs === lastScanMtimeMs && byId.size > 0) return; // 无变化
85
+ lastScanMtimeMs = dirStat.mtimeMs;
86
+
87
+ byId.clear();
88
+ let entries;
89
+ try {
90
+ entries = await fs.readdir(capsDir, { withFileTypes: true });
91
+ } catch {
92
+ return;
93
+ }
94
+ for (const entry of entries) {
95
+ if (!entry.isDirectory()) continue;
96
+ if (!CAPABILITY_IDS.includes(entry.name)) continue; // 白名单外不加载(未知能力不静默启用)
97
+ const source = path.join(capsDir, entry.name, 'index.js');
98
+ try {
99
+ const module = await import(`${source}?t=${Date.now()}`);
100
+ const cap = module.default;
101
+ const verdict = validateCapability(cap, source);
102
+ if (!verdict.ok) {
103
+ loadFailures.push(verdict.reason);
104
+ logger.warn('capability plugin rejected', { reason: verdict.reason });
105
+ continue;
106
+ }
107
+ // 对账:appliesTo 里的格式如果已知名单里没有,记告警(格式包可能还没装)
108
+ for (const formatId of cap.appliesTo) {
109
+ if (knownFormatIds.size > 0 && !knownFormatIds.has(formatId)) {
110
+ logger.warn('能力 appliesTo 指向未知格式(对账告警,不阻止加载)', { capability: cap.id, formatId });
111
+ }
112
+ }
113
+ byId.set(cap.id, cap);
114
+ } catch (error) {
115
+ loadFailures.push(`${source}: ${error.message}`);
116
+ logger.warn('capability plugin rejected', { reason: `${source}: ${error.message}` });
117
+ }
118
+ }
119
+ if (loadFailures.length > 0) logger.warn('capability registry load failures', { count: loadFailures.length });
120
+ }
121
+
122
+ /**
123
+ * 落盘脱敏(PW-1):遍历任务 options.capabilities,用各能力的 redact() 剔除 secret
124
+ * 字段(明文永不落 tasks.json)。manager 不认识具体能力,由 registry 提供通用剥离器。
125
+ * 能力未加载但参数还在(重启后重试场景)→ 保留方法名等非敏感字段,剔除 password。
126
+ */
127
+ function sanitizeOptionsForPersist(options) {
128
+ if (!options) return options;
129
+ // P4:解压侧的顶层 options.password 同样是明文(PW-1)——落盘时替换为 hasPassword 布尔
130
+ const hasTopPassword = typeof options.password === 'string' && options.password.length > 0;
131
+ const caps = options.capabilities;
132
+ if (!caps || typeof caps !== 'object') {
133
+ if (!hasTopPassword) return options;
134
+ const { password: _removed, ...rest } = options;
135
+ return { ...rest, hasPassword: true };
136
+ }
137
+ const sanitized = { ...options, capabilities: {} };
138
+ for (const [capId, params] of Object.entries(caps)) {
139
+ const cap = byId.get(capId);
140
+ if (cap && typeof cap.redact === 'function') {
141
+ sanitized.capabilities[capId] = cap.redact(params);
142
+ continue;
143
+ }
144
+ const filtered = {};
145
+ for (const [key, value] of Object.entries(params || {})) {
146
+ const isSecret = Boolean(value) && (key === 'password' || key === 'value');
147
+ if (!isSecret) filtered[key] = value;
148
+ }
149
+ sanitized.capabilities[capId] = filtered;
150
+ }
151
+ if (hasTopPassword) {
152
+ delete sanitized.password;
153
+ sanitized.hasPassword = true;
154
+ }
155
+ return sanitized;
156
+ }
157
+
158
+ return {
159
+ /** 启动/变更时重扫(幂等) */
160
+ rescan,
161
+ /** 查某格式启用的能力集合(已加载 ∩ appliesTo;格式自己的 capabilities 声明由调用方用 checkFormatCapabilities 校验) */
162
+ capabilitiesForFormat,
163
+ checkFormatCapabilities,
164
+ /** PW-1:任务落盘前的通用脱敏器(剔除 capabilities 里的 secret 字段) */
165
+ sanitizeOptionsForPersist,
166
+ /** 能力元数据(给 GET /capabilities/:id:脱敏字段描述 + methods 动态列表) */
167
+ get: (id) => byId.get(id) || null,
168
+ list: () => [...byId.values()],
169
+ loadFailures: () => [...loadFailures],
170
+ };
171
+ }