paperdepot 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +12 -5
- package/README.md +25 -22
- package/lib/client.js +663 -179
- package/lib/impl.js +65 -0
- package/package.json +1 -1
- package/pipeline/config/tagmap_/344/273/252/345/231/250.json +554 -554
- package/pipeline/config/tagmap_/345/272/224/347/224/250.json +626 -626
- package/pipeline/config/tagmap_/346/226/271/346/263/225.json +890 -890
- package/pipeline/config/tagmap_/351/242/206/345/237/237.json +221 -221
- package/pipeline/tools/mineru_parse.py +462 -462
package/lib/impl.js
CHANGED
|
@@ -524,6 +524,20 @@ function canonicalTags(cfg) {
|
|
|
524
524
|
return out
|
|
525
525
|
}
|
|
526
526
|
|
|
527
|
+
/** 待生成摘要的条目(index.md 的「一句话总结」还是占位符)。 */
|
|
528
|
+
function listPendingDirs(vault) {
|
|
529
|
+
const dir = path.join(vault, '1-Papers')
|
|
530
|
+
const out = []
|
|
531
|
+
for (const name of listDirs(dir).filter((d) => !d.startsWith('_'))) {
|
|
532
|
+
try {
|
|
533
|
+
if (parseEntry(path.join(dir, name, 'index.md')).pending) out.push(name)
|
|
534
|
+
} catch {
|
|
535
|
+
/* 坏条目跳过,不阻塞批量 */
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
return out
|
|
539
|
+
}
|
|
540
|
+
|
|
527
541
|
async function summarizeOne(job, cfg, host, dir) {
|
|
528
542
|
if (!fs.existsSync(path.join(cfg.vault, '1-Papers', dir, 'index.md'))) {
|
|
529
543
|
throw new Error('条目目录不存在或缺少 index.md:' + dir)
|
|
@@ -1135,6 +1149,57 @@ export function createHandler(cfg, host) {
|
|
|
1135
1149
|
})()
|
|
1136
1150
|
return sendJson(res, 202, { ok: true, accepted: true, dir: name })
|
|
1137
1151
|
}
|
|
1152
|
+
// 批量补摘要:把「待生成」条目排队跑完(顺序执行,可停止,再跑自动跳过已完成的)
|
|
1153
|
+
if (p === '/paperdepot/api/summarize-pending' && req.method === 'POST') {
|
|
1154
|
+
if (jobState().job && jobState().job.state === 'running') return sendJson(res, 409, { ok: false, error: '已有任务在跑,等它结束' })
|
|
1155
|
+
const limit = Math.min(Number(url.searchParams.get('limit')) || 0, 500)
|
|
1156
|
+
const all = listPendingDirs(config.vault)
|
|
1157
|
+
const queue = limit > 0 ? all.slice(0, limit) : all
|
|
1158
|
+
if (!queue.length) return sendJson(res, 400, { ok: false, error: '没有待生成摘要的条目' })
|
|
1159
|
+
const job = {
|
|
1160
|
+
id: Date.now().toString(36),
|
|
1161
|
+
kind: 'summarize-batch',
|
|
1162
|
+
file: '',
|
|
1163
|
+
state: 'running',
|
|
1164
|
+
total: queue.length,
|
|
1165
|
+
done: 0,
|
|
1166
|
+
failed: [],
|
|
1167
|
+
remaining: Math.max(0, all.length - queue.length),
|
|
1168
|
+
cancelRequested: false,
|
|
1169
|
+
lines: [`▶ 批量补摘要:排队 ${queue.length} 篇(库里待生成 ${all.length} 篇)`, ` 模型: ${config.provider}/${config.model}`],
|
|
1170
|
+
startedAt: new Date().toISOString(),
|
|
1171
|
+
}
|
|
1172
|
+
jobState().job = job
|
|
1173
|
+
;(async () => {
|
|
1174
|
+
for (let i = 0; i < queue.length; i++) {
|
|
1175
|
+
if (job.cancelRequested) break
|
|
1176
|
+
const name = queue[i]
|
|
1177
|
+
job.done = i
|
|
1178
|
+
job.lines.push('', `[${i + 1}/${queue.length}] ▶ ${name}`)
|
|
1179
|
+
try {
|
|
1180
|
+
await summarizeOne(job, config, host, name)
|
|
1181
|
+
job.done = i + 1
|
|
1182
|
+
} catch (err) {
|
|
1183
|
+
job.done = i + 1
|
|
1184
|
+
job.failed.push({ dir: name, error: String((err && err.message) || err) })
|
|
1185
|
+
job.lines.push(`❌ 跳过(可稍后单独重跑):${String((err && err.message) || err)}`)
|
|
1186
|
+
}
|
|
1187
|
+
trimLog(job)
|
|
1188
|
+
}
|
|
1189
|
+
job.state = job.cancelRequested ? 'cancelled' : 'done'
|
|
1190
|
+
job.finishedAt = new Date().toISOString()
|
|
1191
|
+
job.lines.push('', `— 结束:成功 ${job.done - job.failed.length}/${job.total},失败 ${job.failed.length}${job.cancelRequested ? '(已手动停止)' : ''} —`)
|
|
1192
|
+
statsCache = { at: 0, vault: '', data: null }
|
|
1193
|
+
})()
|
|
1194
|
+
return sendJson(res, 202, { ok: true, accepted: true, total: queue.length, pending: all.length, remaining: job.remaining })
|
|
1195
|
+
}
|
|
1196
|
+
// 停止当前任务(批量补摘要跑长任务时用;下次再点会从剩下的条目继续)
|
|
1197
|
+
if (p === '/paperdepot/api/job-cancel' && req.method === 'POST') {
|
|
1198
|
+
const job = jobState().job
|
|
1199
|
+
if (!job || job.state !== 'running') return sendJson(res, 400, { ok: false, error: '没有在跑的任务' })
|
|
1200
|
+
job.cancelRequested = true
|
|
1201
|
+
return sendJson(res, 200, { ok: true, accepted: true })
|
|
1202
|
+
}
|
|
1138
1203
|
return sendJson(res, 404, { ok: false, error: 'not found', path: p })
|
|
1139
1204
|
} catch (err) {
|
|
1140
1205
|
return sendJson(res, 500, { ok: false, error: String((err && err.message) || err) })
|