qiyu-cli 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +464 -2
- package/dist/cli.js +248 -0
- package/dist/commands/api.js +770 -0
- package/dist/commands/auth.js +349 -0
- package/dist/commands/credentials-source.js +70 -0
- package/dist/commands/db.js +503 -0
- package/dist/commands/local-fallback-wiring.js +67 -0
- package/dist/commands/local-fallback.js +34 -0
- package/dist/commands/query.js +564 -0
- package/dist/commands/skill-specs.js +64 -0
- package/dist/commands/skill.js +863 -0
- package/dist/commands/spec-types.js +145 -0
- package/dist/commands/stats.js +299 -0
- package/dist/commands/store-flags.js +56 -0
- package/dist/commands/store.js +258 -0
- package/dist/commands/sync-run.js +479 -0
- package/dist/commands/sync.js +757 -0
- package/dist/commands/tree.js +27 -0
- package/dist/core/config.js +259 -0
- package/dist/core/credentials.js +71 -0
- package/dist/core/edition.js +18 -0
- package/dist/core/envelope.js +201 -0
- package/dist/core/errors.js +356 -0
- package/dist/core/http.js +348 -0
- package/dist/core/ratelimit.js +126 -0
- package/dist/core/registry/index.js +212 -0
- package/dist/core/registry/params/ai.knowledge.addKnowledge.json +88 -0
- package/dist/core/registry/params/ai.knowledge.deleteKnowledges.json +46 -0
- package/dist/core/registry/params/ai.knowledge.editChannelAnswer.json +60 -0
- package/dist/core/registry/params/ai.knowledge.editKnowledge.json +95 -0
- package/dist/core/registry/params/ai.knowledge.editKnowledgeNoSimilars.json +102 -0
- package/dist/core/registry/params/ai.knowledge.getCategoryTree.json +39 -0
- package/dist/core/registry/params/ai.knowledge.getDetail.json +46 -0
- package/dist/core/registry/params/ai.knowledge.search.json +67 -0
- package/dist/core/registry/params/ai.statistic.transferHuman.json +60 -0
- package/dist/core/registry/params/category.list.json +39 -0
- package/dist/core/registry/params/chat.queue.group.query.json +39 -0
- package/dist/core/registry/params/crm.modifyOrder.json +48 -0
- package/dist/core/registry/params/crm.modifyUser.json +48 -0
- package/dist/core/registry/params/crm.syncCrmInfo.json +46 -0
- package/dist/core/registry/params/crm.tagChange.json +74 -0
- package/dist/core/registry/params/data.overview.ipcc.json +32 -0
- package/dist/core/registry/params/data.overview.session.json +53 -0
- package/dist/core/registry/params/evaluation.setting.get.json +39 -0
- package/dist/core/registry/params/event.applyStaff.json +116 -0
- package/dist/core/registry/params/event.evaluate.json +74 -0
- package/dist/core/registry/params/event.inviteEvaluate.json +81 -0
- package/dist/core/registry/params/event.qcResult.json +166 -0
- package/dist/core/registry/params/event.queryQueueStatus.json +39 -0
- package/dist/core/registry/params/event.queueEnter.json +40 -0
- package/dist/core/registry/params/event.queueLeave.json +46 -0
- package/dist/core/registry/params/event.quitQueue.json +39 -0
- package/dist/core/registry/params/event.sessionChange.json +95 -0
- package/dist/core/registry/params/event.sessionData.json +452 -0
- package/dist/core/registry/params/event.sessionEnd.json +117 -0
- package/dist/core/registry/params/event.sessionStart.json +109 -0
- package/dist/core/registry/params/event.updateUInfo.json +46 -0
- package/dist/core/registry/params/export.session.check.json +7 -0
- package/dist/core/registry/params/export.session.json +7 -0
- package/dist/core/registry/params/export.session.one.json +7 -0
- package/dist/core/registry/params/export.session.one.message.json +7 -0
- package/dist/core/registry/params/get_call_order.json +50 -0
- package/dist/core/registry/params/get_call_user_info.json +36 -0
- package/dist/core/registry/params/get_order.json +50 -0
- package/dist/core/registry/params/get_token.json +22 -0
- package/dist/core/registry/params/get_user_info.json +36 -0
- package/dist/core/registry/params/icoutcall.callIntercept.list.json +32 -0
- package/dist/core/registry/params/icoutcall.dialogFlow.fuzzySearch.json +53 -0
- package/dist/core/registry/params/icoutcall.dialogFlow.getDialogFlowList.json +32 -0
- package/dist/core/registry/params/icoutcall.get.customer.json +39 -0
- package/dist/core/registry/params/icoutcall.get.jobdetail.json +39 -0
- package/dist/core/registry/params/icoutcall.get.jobproperties.json +39 -0
- package/dist/core/registry/params/icoutcall.history.seat.json +67 -0
- package/dist/core/registry/params/icoutcall.history.team.json +67 -0
- package/dist/core/registry/params/icoutcall.job.create.json +53 -0
- package/dist/core/registry/params/icoutcall.job.importcustomer.json +46 -0
- package/dist/core/registry/params/icoutcall.job.pause.json +39 -0
- package/dist/core/registry/params/icoutcall.job.readdcustomer.json +109 -0
- package/dist/core/registry/params/icoutcall.job.start.json +39 -0
- package/dist/core/registry/params/icoutcall.job.terminate.json +39 -0
- package/dist/core/registry/params/icoutcall.phone.getLineList.json +32 -0
- package/dist/core/registry/params/ipcc.calltask.add.auto.json +117 -0
- package/dist/core/registry/params/ipcc.calltask.add.byCustom.json +96 -0
- package/dist/core/registry/params/ipcc.calltask.add.byRelation.json +82 -0
- package/dist/core/registry/params/ipcc.calltask.add.toGroup.json +89 -0
- package/dist/core/registry/params/ipcc.calltask.add.toStaff.json +89 -0
- package/dist/core/registry/params/ipcc.calltask.get.json +47 -0
- package/dist/core/registry/params/ipcc.calltaskdetail.add.json +54 -0
- package/dist/core/registry/params/ipcc.calltaskdetail.delete.json +40 -0
- package/dist/core/registry/params/ipcc.calltaskdetail.get.json +54 -0
- package/dist/core/registry/params/ipcc.calltaskdetail.update.json +40 -0
- package/dist/core/registry/params/ipcc.customfield.template.get.json +47 -0
- package/dist/core/registry/params/ipcc.login.json +47 -0
- package/dist/core/registry/params/kefu.add.json +130 -0
- package/dist/core/registry/params/kefu.delete.json +39 -0
- package/dist/core/registry/params/kefu.update.json +137 -0
- package/dist/core/registry/params/kefu.v2.query.json +39 -0
- package/dist/core/registry/params/kefuGroup.add.json +39 -0
- package/dist/core/registry/params/kefuGroup.delete.json +39 -0
- package/dist/core/registry/params/kefuGroup.update.json +46 -0
- package/dist/core/registry/params/kefuGroup.updateKefus.json +46 -0
- package/dist/core/registry/params/merchant.account.login.json +53 -0
- package/dist/core/registry/params/merchant.changePassword.json +46 -0
- package/dist/core/registry/params/merchant.create.json +81 -0
- package/dist/core/registry/params/merchant.disable.json +39 -0
- package/dist/core/registry/params/merchant.enable.json +39 -0
- package/dist/core/registry/params/merchant.list.json +46 -0
- package/dist/core/registry/params/merchant.query.json +46 -0
- package/dist/core/registry/params/merchant.update.json +74 -0
- package/dist/core/registry/params/message.send.image.json +53 -0
- package/dist/core/registry/params/message.send.json +53 -0
- package/dist/core/registry/params/message.send.voice.json +53 -0
- package/dist/core/registry/params/message.sendFile.json +39 -0
- package/dist/core/registry/params/message.uploadFile.json +39 -0
- package/dist/core/registry/params/online.session.category.list.json +39 -0
- package/dist/core/registry/params/online.session.customfield.template.json +39 -0
- package/dist/core/registry/params/online.session.customfield.update.json +74 -0
- package/dist/core/registry/params/order.upload.json +40 -0
- package/dist/core/registry/params/push.msg.image.json +74 -0
- package/dist/core/registry/params/push.msg.richText.json +74 -0
- package/dist/core/registry/params/push.msg.text.json +74 -0
- package/dist/core/registry/params/receive.message.json +89 -0
- package/dist/core/registry/params/robot.data.getQuestionsByCategoryName.json +67 -0
- package/dist/core/registry/params/robot.data.knowledge.json +46 -0
- package/dist/core/registry/params/robot.data.kpoint.json +46 -0
- package/dist/core/registry/params/robot.data.queryStandardByEffectiveQuestion.json +46 -0
- package/dist/core/registry/params/robot.data.queryStandardQuestionList.json +46 -0
- package/dist/core/registry/params/robot.evaluation.comment.json +67 -0
- package/dist/core/registry/params/robot.evaluation.setting.json +46 -0
- package/dist/core/registry/params/robot.session.assist.json +74 -0
- package/dist/core/registry/params/robot.session.chat.json +74 -0
- package/dist/core/registry/params/robot.session.close.json +53 -0
- package/dist/core/registry/params/robot.session.commonQuestions.json +74 -0
- package/dist/core/registry/params/robot.session.create.json +81 -0
- package/dist/core/registry/params/robot.session.evaluation.json +67 -0
- package/dist/core/registry/params/robot.session.msgAssociate.json +46 -0
- package/dist/core/registry/params/robot.session.msgEvaluation.json +53 -0
- package/dist/core/registry/params/robot.session.msgPredict.json +74 -0
- package/dist/core/registry/params/robot.session.transfer.json +36 -0
- package/dist/core/registry/params/robot.session.uploadFile.json +32 -0
- package/dist/core/registry/params/robot.statistic.summary.json +53 -0
- package/dist/core/registry/params/robot.transfer.config.json +15 -0
- package/dist/core/registry/params/session.user.profile.json +39 -0
- package/dist/core/registry/params/settings.wxTemplates.json +46 -0
- package/dist/core/registry/params/smstask.create.json +53 -0
- package/dist/core/registry/params/smstask.status.json +39 -0
- package/dist/core/registry/params/staff.group.list.new.json +39 -0
- package/dist/core/registry/params/staff.group.members.new.json +46 -0
- package/dist/core/registry/params/staff.list.new.json +46 -0
- package/dist/core/registry/params/staff.login.json +47 -0
- package/dist/core/registry/params/statistic.overview.json +61 -0
- package/dist/core/registry/params/statistic.performance.json +82 -0
- package/dist/core/registry/params/statistic.salaryDetail.json +54 -0
- package/dist/core/registry/params/statistic.satisfaction.report.json +68 -0
- package/dist/core/registry/params/statistic.staffAttendance.json +75 -0
- package/dist/core/registry/params/statistic.staffquality.json +75 -0
- package/dist/core/registry/params/statistic.staffworklod.json +68 -0
- package/dist/core/registry/params/tag.all.json +32 -0
- package/dist/core/registry/params/tag.batchUpdate.json +32 -0
- package/dist/core/registry/params/tag.update.json +60 -0
- package/dist/core/registry/params/ticket.apply.json +46 -0
- package/dist/core/registry/params/ticket.batch.modify.json +53 -0
- package/dist/core/registry/params/ticket.create.json +151 -0
- package/dist/core/registry/params/ticket.filter.count.json +46 -0
- package/dist/core/registry/params/ticket.filter.list.json +46 -0
- package/dist/core/registry/params/ticket.finish.json +60 -0
- package/dist/core/registry/params/ticket.followers.update.json +60 -0
- package/dist/core/registry/params/ticket.list.json +74 -0
- package/dist/core/registry/params/ticket.log.json +39 -0
- package/dist/core/registry/params/ticket.modify.json +74 -0
- package/dist/core/registry/params/ticket.new.detail.json +39 -0
- package/dist/core/registry/params/ticket.reminder.json +60 -0
- package/dist/core/registry/params/ticket.reopen.json +67 -0
- package/dist/core/registry/params/ticket.reply.json +60 -0
- package/dist/core/registry/params/ticket.search.json +116 -0
- package/dist/core/registry/params/ticket.template.fields.json +39 -0
- package/dist/core/registry/params/ticket.template.list.json +39 -0
- package/dist/core/registry/params/ticket.transfer.json +74 -0
- package/dist/core/registry/params/track.event.set.json +32 -0
- package/dist/core/registry/params/wecom.robot.switch.json +43 -0
- package/dist/core/registry/params.js +84 -0
- package/dist/core/registry/specs.json +2172 -0
- package/dist/core/registry/validate.json +11 -0
- package/dist/core/registry/validation-overrides.js +98 -0
- package/dist/core/sanitize.js +24 -0
- package/dist/core/sign.js +21 -0
- package/dist/core/time.js +194 -0
- package/dist/core/validate.js +140 -0
- package/dist/output.js +263 -0
- package/dist/postinstall.js +178 -0
- package/dist/query/errors.js +395 -0
- package/dist/query/mask.js +368 -0
- package/dist/query/metric-catalog.js +85 -0
- package/dist/query/sandbox.js +544 -0
- package/dist/query/table-query.js +297 -0
- package/dist/query/time-shape.js +44 -0
- package/dist/schedule/install.js +377 -0
- package/dist/skill/check.js +429 -0
- package/dist/skill/detect.js +336 -0
- package/dist/skill/install.js +1202 -0
- package/dist/skill/paths.js +343 -0
- package/dist/skill/qiyu/SKILL.md +86 -0
- package/dist/skill/qiyu/errors.md +55 -0
- package/dist/skill/qiyu/forbidden.md +52 -0
- package/dist/skill/qiyu/metrics.md +65 -0
- package/dist/skill/qiyu/protocols.md +55 -0
- package/dist/skill/qiyu/setup.md +50 -0
- package/dist/skill/refs.js +379 -0
- package/dist/store/config.js +137 -0
- package/dist/store/duckdb.js +321 -0
- package/dist/store/index.js +18 -0
- package/dist/store/retry.js +36 -0
- package/dist/store/schema.js +419 -0
- package/dist/store/types.js +4 -0
- package/dist/sync/day.js +8 -0
- package/dist/sync/engine.js +818 -0
- package/dist/sync/rate-gate.js +95 -0
- package/dist/sync/registry.js +22 -0
- package/dist/sync/state.js +114 -0
- package/dist/sync/tasks/dim.js +186 -0
- package/dist/sync/tasks/knowledge.js +252 -0
- package/dist/sync/tasks/pace.js +66 -0
- package/dist/sync/tasks/pipeline.js +321 -0
- package/dist/sync/tasks/reports.js +448 -0
- package/dist/sync/tasks/sessions.js +188 -0
- package/dist/sync/tasks/tickets.js +219 -0
- package/dist/sync/types.js +1 -0
- package/dist/sync/windowing.js +56 -0
- package/package.json +52 -4
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
// src/sync/tasks/knowledge.ts
|
|
2
|
+
// D-T11:知识库 3 快照任务(spec §5.4,设计稿 §10.5 实测事实):
|
|
3
|
+
// ① robot.data.knowledge / robot.data.kpoint 是**游标分页**:mid = 上一页末行 id
|
|
4
|
+
// (严格大于)、size ≤ 1000、isEnd=1 终止;信封形状 C——message 是 JSON 字符串,
|
|
5
|
+
// 归一化在 core/envelope 层完成(payload 即解析物),任务代码不二次猜测;
|
|
6
|
+
// ② **robot.data.knowledge 的 robotId 被服务端静默忽略**(实测 A/B:传与不传 total
|
|
7
|
+
// 相同)——静默丢弃比报错更危险(看起来像过滤生效),含 robotId 的参数在构造
|
|
8
|
+
// body 前客户端 usage 拒绝、绝不透传(契约诚实性);
|
|
9
|
+
// ③ knowledge_items 是全部机器人的**并集**;knowledge_categories **按 robotId 隔离**
|
|
10
|
+
// (实测 5 个 robotId 得到 5 棵不同的树)——两接口语义不同,谁也不能从对方推断。
|
|
11
|
+
// categories 的驱动序只能来自 knowledge_items 落库后的 DISTINCT robot_id
|
|
12
|
+
// (robot.data.knowledge 响应含 robotId,是全量遍历下唯一的 API 侧发现途径);
|
|
13
|
+
// ④ getCategoryTree 空树机器人返回 {code:200,message:'success'} 且**无 data 键**
|
|
14
|
+
// (形状 A not_found)——是「知识库无类目」的合法状态,0 行、不报错;
|
|
15
|
+
// ⑤ 计数红线:**测试断言不变量,不硬编码实测行数**(12778 等计数会漂移)。
|
|
16
|
+
// 全部 kind=snapshot:幂等来自引擎快照路径的全删重插(engine.ts runSnapshot)。
|
|
17
|
+
import { callPaced, nowIsoOf } from './pace.js';
|
|
18
|
+
/** 单页上限(文档与实测一致:size 一次最多 1000)。 */
|
|
19
|
+
const PAGE_SIZE = 1000;
|
|
20
|
+
/** 限频节流与测试 sleep 注入收敛至共享 pace.ts(中-1)。 */
|
|
21
|
+
export { __setSleep } from './pace.js';
|
|
22
|
+
/** usage 错误(退出码 2):本地参数问题,重试无用(形态与 tickets.ts 一致)。 */
|
|
23
|
+
function usageError(msg, hint) {
|
|
24
|
+
const e = new Error(msg);
|
|
25
|
+
e.kind = 'usage';
|
|
26
|
+
e.hint = hint;
|
|
27
|
+
return e;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* robotId 透传守卫:robot.data.knowledge / robot.data.kpoint 的 robotId 被服务端
|
|
31
|
+
* **静默忽略**——任务 fetch 收到含 robotId 的参数必须 usage 拒绝(绝不发请求)。
|
|
32
|
+
* ⚠️ 只有这两个 items/points 接口如此;getCategoryTree 的 robotId 是生效参数
|
|
33
|
+
*(按机器人隔离),由 categories 任务正常使用。
|
|
34
|
+
*/
|
|
35
|
+
function assertNoRobotId(w, taskId) {
|
|
36
|
+
if (w !== null && typeof w === 'object' && 'robotId' in w) {
|
|
37
|
+
throw usageError(`${taskId} 不接受 robotId:robot.data 系列接口会静默忽略它(返回企业全量而非该机器人的数据),` +
|
|
38
|
+
'传了只会误导调用方。按机器人过滤请用 knowledge.categories(getCategoryTree 按 robotId 隔离)。', '这是同步任务的本地参数问题(robotId 被服务端静默忽略),删除该参数后重跑');
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* 游标分页拉取形状 C 载荷({total, data[], isEnd}),带 total 核对:
|
|
43
|
+
* · mid 从 0 起,每页后 = 本页末行 id(服务端语义:下一批 **严格大于** mid);
|
|
44
|
+
* · isEnd=1 终止;size 恒 1000;
|
|
45
|
+
* · 空页但 isEnd=0 → 终止循环并显式失败(mid 已无可推进依据,继续翻只会死循环);
|
|
46
|
+
* · **翻页上限保护**:maxPages = ceil(total/size) + 5 余量(total 未知时保守 1000 页)
|
|
47
|
+
* ——非空页但 isEnd 恒 0、或末行 id 缺失致 mid=NaN 原地打转的响应形态漂移,
|
|
48
|
+
* 都在有限次调用后显式失败,绝不挂起;
|
|
49
|
+
* · 各页声明 total 必须一致、结束时累计行数必须等于 total——「看似成功的短读」
|
|
50
|
+
* 是最危险的静默失败方向(配对核对的前置事实检查)。
|
|
51
|
+
*/
|
|
52
|
+
async function pagedCursor(endpointId, ctx) {
|
|
53
|
+
const out = [];
|
|
54
|
+
let mid = 0;
|
|
55
|
+
let declared = null;
|
|
56
|
+
let page = 0;
|
|
57
|
+
for (;;) {
|
|
58
|
+
// 上限在发请求前判定:声明 total 已知 → ceil(total/size)+5;未知 → 保守 1000 页
|
|
59
|
+
const maxPages = declared !== null
|
|
60
|
+
? Math.ceil(declared / PAGE_SIZE) + 5
|
|
61
|
+
: 1000;
|
|
62
|
+
if (page >= maxPages) {
|
|
63
|
+
throw new Error(`接口 ${endpointId} 翻页超限(${page} 页 > 上限 ${maxPages},声明 total=` +
|
|
64
|
+
`${String(declared)},已取 ${out.length} 行):isEnd 恒 0 或游标未推进(响应形态漂移),拒绝落库`);
|
|
65
|
+
}
|
|
66
|
+
page++;
|
|
67
|
+
// B 中-6:callPaced 的键就是 endpoint id(闸门按路径拦)——任务 id 不参与。
|
|
68
|
+
const res = await callPaced(endpointId, { mid, size: PAGE_SIZE }, ctx);
|
|
69
|
+
const p = res.payload;
|
|
70
|
+
if (p === null || typeof p !== 'object' || !Array.isArray(p.data)) {
|
|
71
|
+
throw new Error(`接口 ${endpointId} 载荷缺 data 数组(响应形态漂移?),拒绝落库`);
|
|
72
|
+
}
|
|
73
|
+
const rows = p.data;
|
|
74
|
+
out.push(...rows);
|
|
75
|
+
if (typeof p.total === 'number') {
|
|
76
|
+
if (declared === null)
|
|
77
|
+
declared = p.total;
|
|
78
|
+
else if (p.total !== declared) {
|
|
79
|
+
throw new Error(`接口 ${endpointId} 各页声明 total 漂移(${declared} → ${p.total}):` +
|
|
80
|
+
'遍历期间上游数据变更,拒绝落库(重跑即可)');
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
if (rows.length > 0)
|
|
84
|
+
mid = Number(rows[rows.length - 1].id);
|
|
85
|
+
const isEnd = p.isEnd === 1;
|
|
86
|
+
// 空页却未到末页:mid 已无可推进依据,继续翻页只会死循环——显式失败(形态漂移)
|
|
87
|
+
if (rows.length === 0 && !isEnd) {
|
|
88
|
+
throw new Error(`接口 ${endpointId} 返回空页但 isEnd=0(声明 total=${String(p.total)},` +
|
|
89
|
+
`已取 ${out.length} 行):响应形态漂移,拒绝落库`);
|
|
90
|
+
}
|
|
91
|
+
if (isEnd) {
|
|
92
|
+
if (declared !== null && out.length !== declared) {
|
|
93
|
+
throw new Error(`接口 ${endpointId} 遍历结束行数不符:声明 total=${declared},实取 ${out.length} 行` +
|
|
94
|
+
'(响应形态漂移/遍历期间上游变更),拒绝落库——重跑即可');
|
|
95
|
+
}
|
|
96
|
+
break;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
return out;
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* 脏行守卫 + 键列提取(按映射表投影)。缺键(undefined)的行丢弃——脏单行不致断流
|
|
103
|
+
* 的统一取舍;**键存在但值为 null 是合法数据**(pointId 仅 37.6% 行有值、robot_name
|
|
104
|
+
* 可空——设计稿 §10.5 实测),落库为 null,绝不因此丢行。
|
|
105
|
+
* ⚠️ INTEGER 列(numericCols 中的)**数字直通**、不做 JSON.stringify——否则 number 1
|
|
106
|
+
* 会被字符串化成 "1" 落 INTEGER 列(依赖 DuckDB 隐式转型是脆弱行为);VARCHAR 列的
|
|
107
|
+
* 非字符串值(ID 是 BIGINT)才字符串化(BIGINT 精度红线)。
|
|
108
|
+
* 返回 `{ rows, dropped }`(修复批 4 #3)——快照任务全删重插,被丢弃的行从表里消失
|
|
109
|
+
* 且无任何痕迹;dropped > 0 时调用方必须用 gapHint 把这件事记进 sync_gaps(口径同
|
|
110
|
+
* dim.staff_group_members 的 C-14),绝不静默丢行。
|
|
111
|
+
*/
|
|
112
|
+
function projectRows(rows, keys, numericCols, nowIso) {
|
|
113
|
+
const out = [];
|
|
114
|
+
let dropped = 0;
|
|
115
|
+
for (const r of rows) {
|
|
116
|
+
const mapped = {};
|
|
117
|
+
let ok = true;
|
|
118
|
+
for (const [col, get] of Object.entries(keys)) {
|
|
119
|
+
const v = get(r); // 只取一次(键缺失判别与取值同一来源)
|
|
120
|
+
if (v === undefined) {
|
|
121
|
+
ok = false;
|
|
122
|
+
break;
|
|
123
|
+
} // 键缺失 → 脏行;null 是合法值
|
|
124
|
+
mapped[col] = v === null ? null
|
|
125
|
+
: numericCols.has(col) ? Number(v)
|
|
126
|
+
: typeof v === 'string' ? v : JSON.stringify(v);
|
|
127
|
+
}
|
|
128
|
+
if (!ok) {
|
|
129
|
+
dropped++;
|
|
130
|
+
continue;
|
|
131
|
+
} // 键缺失行:丢弃且不计入 fetched(与 apiCount 口径一致)
|
|
132
|
+
// 其余字段整体收敛进 payload(22 键对 16 键文档漂移——payload 保真全部业务字段)
|
|
133
|
+
out.push({ ...mapped, payload: JSON.stringify(r), synchronized_at: nowIso });
|
|
134
|
+
}
|
|
135
|
+
return { rows: out, dropped };
|
|
136
|
+
}
|
|
137
|
+
/** 类目树拍平(children 在叶子上为 null,实测形态——空数组/null 都没有子节点)。 */
|
|
138
|
+
function flattenTree(robotId, nodes, level, parentId, out, nowIso) {
|
|
139
|
+
if (!Array.isArray(nodes))
|
|
140
|
+
return;
|
|
141
|
+
for (const n of nodes) {
|
|
142
|
+
const cid = n.id;
|
|
143
|
+
if (cid === null || cid === undefined)
|
|
144
|
+
continue; // 缺 id 的脏节点:丢弃
|
|
145
|
+
const sId = String(cid);
|
|
146
|
+
out.push({ robot_id: robotId, category_id: sId, name: typeof n.name === 'string' ? n.name : null,
|
|
147
|
+
parent_id: parentId, level, payload: JSON.stringify(n), synchronized_at: nowIso });
|
|
148
|
+
flattenTree(robotId, n.children, level + 1, sId, out, nowIso);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* 类目树拍平的共享入口(T13 接线):从根(level=1)拍平一棵 getCategoryTree 响应。
|
|
153
|
+
* 编排层的 robotId 注入路径消费同一实现——复制投影会让行形态在两处漂移。
|
|
154
|
+
*/
|
|
155
|
+
export function flattenCategoryTree(robotId, payload, out, nowIso) {
|
|
156
|
+
flattenTree(robotId, payload, 1, null, out, nowIso);
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* knowledge_items / knowledge_points 共用的游标快照 fetch。numericCols 是 INTEGER 列集(数字直通)。
|
|
160
|
+
* 修复批 4 #3:projectRows 丢弃的缺键行在快照(全删重插)下**从表里消失**,而
|
|
161
|
+
* `apiCount: mapped.length` 让配对核对恒等(丢行不留痕)→ dropped > 0 时显式记 gapHint。
|
|
162
|
+
*/
|
|
163
|
+
function fetchCursor(taskId, endpointId, keys, numericCols = []) {
|
|
164
|
+
const numeric = new Set(numericCols);
|
|
165
|
+
return async (w, ctx) => {
|
|
166
|
+
assertNoRobotId(w, taskId);
|
|
167
|
+
const nowIso = nowIsoOf(ctx);
|
|
168
|
+
const rows = await pagedCursor(endpointId, ctx);
|
|
169
|
+
const { rows: mapped, dropped } = projectRows(rows, keys, numeric, nowIso);
|
|
170
|
+
return {
|
|
171
|
+
rows: mapped, apiCount: mapped.length,
|
|
172
|
+
...(dropped > 0
|
|
173
|
+
? {
|
|
174
|
+
gapHint: {
|
|
175
|
+
reason: 'knowledge_row_malformed',
|
|
176
|
+
detail: `${taskId}: ${dropped} 行缺键列被丢弃(快照全删重插下这些行已消失)——` +
|
|
177
|
+
`接口 ${endpointId} 的响应形态漂移,请核对`,
|
|
178
|
+
},
|
|
179
|
+
}
|
|
180
|
+
: {}),
|
|
181
|
+
};
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
export const KNOWLEDGE_TASKS = [
|
|
185
|
+
{
|
|
186
|
+
id: 'knowledge.items', table: 'knowledge_items', kind: 'snapshot',
|
|
187
|
+
watermark: { field: 'synchronized_at', initial: null },
|
|
188
|
+
primaryKey: ['id'],
|
|
189
|
+
endpoint: 'robot.data.knowledge',
|
|
190
|
+
write: { kind: 'upsertRows' },
|
|
191
|
+
// 企业级全量(全部机器人并集);robotId 被静默忽略 → fetch 守卫拒绝透传。
|
|
192
|
+
// 键列:id/robot_id/robot_name/group_id/ref_id/point_id(groupId 是知识实体真正的
|
|
193
|
+
// 主键概念但全局键仍是上游 id,红线 1);type 是响应里的数字(0/1 枚举语义见设计稿)。
|
|
194
|
+
fetch: fetchCursor('knowledge.items', 'robot.data.knowledge', {
|
|
195
|
+
id: (r) => r.id,
|
|
196
|
+
robot_id: (r) => r.robotId,
|
|
197
|
+
robot_name: (r) => r.robotName,
|
|
198
|
+
type: (r) => r.type,
|
|
199
|
+
group_id: (r) => r.groupId,
|
|
200
|
+
ref_id: (r) => r.refId,
|
|
201
|
+
point_id: (r) => r.pointId,
|
|
202
|
+
}, ['type']),
|
|
203
|
+
},
|
|
204
|
+
{
|
|
205
|
+
id: 'knowledge.points', table: 'knowledge_points', kind: 'snapshot',
|
|
206
|
+
watermark: { field: 'synchronized_at', initial: null },
|
|
207
|
+
primaryKey: ['id'],
|
|
208
|
+
endpoint: 'robot.data.kpoint',
|
|
209
|
+
write: { kind: 'upsertRows' },
|
|
210
|
+
// kpoint 实测全字段存在且无 robotId 键(归属只能经 items.point_id 反推)——
|
|
211
|
+
// 单键列 id 即可(全字段都在 payload 里保真)。
|
|
212
|
+
fetch: fetchCursor('knowledge.points', 'robot.data.kpoint', {
|
|
213
|
+
id: (r) => r.id,
|
|
214
|
+
}),
|
|
215
|
+
},
|
|
216
|
+
{
|
|
217
|
+
id: 'knowledge.categories', table: 'knowledge_categories', kind: 'snapshot',
|
|
218
|
+
watermark: { field: 'synchronized_at', initial: null },
|
|
219
|
+
endpoint: 'ai.knowledge.getCategoryTree',
|
|
220
|
+
// 主键必须含 robot_id(类目 id 空间按机器人划分,§10.5.9——单列主键会跨机器人覆盖)
|
|
221
|
+
primaryKey: ['robot_id', 'category_id'],
|
|
222
|
+
write: { kind: 'upsertRows' },
|
|
223
|
+
// 驱动序来自 knowledge_items 落库后的 DISTINCT robot_id——items 表为空是任务序
|
|
224
|
+
// 错误(先跑 knowledge.items),显式失败而非静默空跑。getCategoryTree 的 robotId
|
|
225
|
+
// 是**生效**参数(按机器人隔离,与 items 语义不同),逐机器人调用;空树机器人
|
|
226
|
+
//(not_found,无 data 键)是合法状态 → 该机器人 0 行、不报错。
|
|
227
|
+
fetch: async (w, ctx) => {
|
|
228
|
+
assertNoRobotId(w, 'knowledge.categories');
|
|
229
|
+
void w;
|
|
230
|
+
const nowIso = nowIsoOf(ctx);
|
|
231
|
+
const r = await ctx.store.query('SELECT DISTINCT robot_id FROM knowledge_items ORDER BY robot_id');
|
|
232
|
+
const robotIds = r.rows.map((x) => String(x.robot_id));
|
|
233
|
+
if (robotIds.length === 0) {
|
|
234
|
+
throw new Error('knowledge_items 表为空:knowledge.categories 按 robotId 隔离驱动,' +
|
|
235
|
+
'请先同步 knowledge.items 任务');
|
|
236
|
+
}
|
|
237
|
+
const out = [];
|
|
238
|
+
for (const rid of robotIds) {
|
|
239
|
+
const res = await callPaced('ai.knowledge.getCategoryTree', { robotId: Number(rid) }, ctx);
|
|
240
|
+
// 形状 A:data 直接是树数组;空树机器人整个丢 data 键(not_found)→ 0 行合法
|
|
241
|
+
if (res.payload === undefined || res.payload === null)
|
|
242
|
+
continue;
|
|
243
|
+
flattenTree(rid, res.payload, 1, null, out, nowIso);
|
|
244
|
+
}
|
|
245
|
+
return { rows: out, apiCount: out.length };
|
|
246
|
+
},
|
|
247
|
+
},
|
|
248
|
+
];
|
|
249
|
+
/** 按 id 取知识库任务(T12 汇总与测试的取用形态)。 */
|
|
250
|
+
export function taskById(id) {
|
|
251
|
+
return KNOWLEDGE_TASKS.find((t) => t.id === id);
|
|
252
|
+
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
// src/sync/tasks/pace.ts
|
|
2
|
+
// 任务层共享的「带冷却重试的调用」+ synchronized_at 盖章时钟(final 审查中-1/低-4)。
|
|
3
|
+
// 此前 reports / knowledge / tickets 三处各自内联同一份 callPaced(同一实现的三个副本,
|
|
4
|
+
// 改动必然漂移)——收敛到本文件,三个任务文件改为复用(__setSleep 经各自模块再导出,
|
|
5
|
+
// 既有测试注入点不变)。
|
|
6
|
+
//
|
|
7
|
+
// 两类限频都在此吸收(「慢不是错」:宁可慢,不超发):
|
|
8
|
+
// · local_rate_limit —— core 进程内令牌桶(core/http.ts)抛出的本地冷却,
|
|
9
|
+
// 按 retryAfter 等待后重试(不写持久化冷却行:那是纯进程内状态);
|
|
10
|
+
// · 远端限频(kind=rate_limit,14009/14500/12001/14402/8011 等)—— 先经持久化
|
|
11
|
+
// 闸门 penalize 写冷却(跨进程生效,下次 acquire 会被拦住),再等待重试。
|
|
12
|
+
// 有界重试(MAX_PACE_ATTEMPTS):冷却行坏数据/服务端持续限频时有限次后显式抛出,
|
|
13
|
+
// 由引擎记 gap,绝不死循环。
|
|
14
|
+
//
|
|
15
|
+
// 全场景审查(修复批 1)修正的两点:
|
|
16
|
+
// · B 高-1:**所有**任务层的调用都必须走这里 —— 裸 `call` 让本地令牌桶的拒绝
|
|
17
|
+
// 直接冒泡成 fetch_failed gap(而它其实只该等一等),远端限频的证据(冷却行)
|
|
18
|
+
// 也没人写。reports 的 fetchSimple、dim 的全部取数、pipeline 的 create/check
|
|
19
|
+
// 已接入。
|
|
20
|
+
// · B 中-6:penalize 的键是 **endpoint id**(不是 task id)—— 闸门按路径拦、
|
|
21
|
+
// 限频表按路径登记,同路径多任务(tickets.search / ticket.logs)必须共享冷却。
|
|
22
|
+
// · B 低-2:等待时长封顶(clampGateWait)—— 冷却行坏数据算出的天文数字不把同步挂死。
|
|
23
|
+
import { call } from '../../core/http.js';
|
|
24
|
+
import { byId } from '../../core/registry/index.js';
|
|
25
|
+
import { clampGateWait } from '../rate-gate.js';
|
|
26
|
+
let sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
27
|
+
/** 仅测试用:替换等待实现(真实同步的等待可达 180 秒,不可测;测试注入零等待)。 */
|
|
28
|
+
export function __setSleep(fn) { sleep = fn; }
|
|
29
|
+
/** 初次 + 3 次等待重试。 */
|
|
30
|
+
const MAX_PACE_ATTEMPTS = 3;
|
|
31
|
+
/**
|
|
32
|
+
* 任务层 synchronized_at 盖章的统一时钟(低-4:此前是裸 `new Date()`,与 ctx.clock
|
|
33
|
+
* 注入面不一致 —— 确定性测试无法断言戳值)。缺省(无 ctx.clock 的直调路径)真实时间。
|
|
34
|
+
*/
|
|
35
|
+
export function nowIsoOf(ctx) {
|
|
36
|
+
return new Date(ctx.clock?.() ?? Date.now()).toISOString();
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* 带冷却重试的调用。
|
|
40
|
+
* ⚠️ `endpointId` 是注册表 id,**也是持久化闸门的键**(B 中-6):penalize 与引擎的
|
|
41
|
+
* `gate.acquire(task.endpoint)` 必须落在同一把键上,冷却才拦得住下一次调用。
|
|
42
|
+
*/
|
|
43
|
+
export async function callPaced(endpointId, body, ctx, attempt = 0) {
|
|
44
|
+
try {
|
|
45
|
+
return await call(byId(endpointId), body, { credentials: ctx.credentials });
|
|
46
|
+
}
|
|
47
|
+
catch (e) {
|
|
48
|
+
const err = e;
|
|
49
|
+
// ⚠️ 本地限频也是 kind=rate_limit(http.ts 在同一处抛出,只有 code 区分)——
|
|
50
|
+
// 必须按 code 分辨,否则会为纯进程内冷却写持久化冷却行(跨进程拦住不该拦的调用)。
|
|
51
|
+
const isLocal = err.code === 'local_rate_limit';
|
|
52
|
+
const isRemote = err.kind === 'rate_limit' && !isLocal;
|
|
53
|
+
if ((!isLocal && !isRemote) || attempt >= MAX_PACE_ATTEMPTS)
|
|
54
|
+
throw e;
|
|
55
|
+
const seconds = err.retryAfter?.seconds ?? 30;
|
|
56
|
+
// 远端限频是证据:写进持久化闸门(闸门不在 ctx —— verify 等直调 fetch 的路径 ——
|
|
57
|
+
// 退化为纯等待,语义仍是「宁可慢」)。键是 endpoint id,同路径任务共享这一行。
|
|
58
|
+
if (isRemote && ctx.gate)
|
|
59
|
+
await ctx.gate.penalize(endpointId, seconds);
|
|
60
|
+
// 等待封顶(B 低-2):限频表最大 180 秒,但坏数据/兜底值不该把单次等待拉成天文数字。
|
|
61
|
+
await sleep(clampGateWait((seconds + 1) * 1000) || GATE_WAIT_MIN_MS);
|
|
62
|
+
return callPaced(endpointId, body, ctx, attempt + 1);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
/** 等待下界:retryAfter 异常(0/NaN)时退化为最小等待,绝不零等待空转。 */
|
|
66
|
+
const GATE_WAIT_MIN_MS = 1_000;
|
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
// src/sync/tasks/pipeline.ts
|
|
2
|
+
// D-T10:会话导出四步流水线(设计稿 §7.5 实测端到端)—— 纯逻辑,副作用全部可注入:
|
|
3
|
+
// 1. create:`export.session`(10 秒 6 次限频,限频表已有该行;进程内令牌桶由 core 层守)
|
|
4
|
+
// body {start, end}(code 可选——实测不校验,不传误导性参数)→ key 在 message
|
|
5
|
+
// 2. poll:`export.session.check` body {key}(code 同样可选)→ 未完成 `14403 Wait...`
|
|
6
|
+
// (core 层判 kind=pending 抛出),完成 `{code:200, message:"<下载链接>"}`;
|
|
7
|
+
// 有界轮询(maxPolls 缺省 120),间隔经 ctx.pollIntervalMs 注入(真实默认 5s)
|
|
8
|
+
// 3. download:ctx.fetchZip(链接) 注入——下载走 ZIP 二进制通道,与 JSON 信封层不同构,
|
|
9
|
+
// 测试 mock 注入合成 zip(生产实现归 T13 接线)
|
|
10
|
+
// 4. unzip + parse:解压密码 = appKey 前 12 字符(ZipCrypto,实测样本验证);
|
|
11
|
+
// session.txt 是 JSONL,**末尾有一行非 JSON 尾部标记 `startTime=...,endTime=...`
|
|
12
|
+
// 必须跳过**(Review Focus 4),空行跳过。该行是**导出自己声明的窗口**(C-8)——
|
|
13
|
+
// 之前它被 `void` 丢弃,现在随返回值带出供任务层与请求窗口核对(见 parseExportFixture)。
|
|
14
|
+
// 去重契约:**按 session.id,绝不能按导出任务 key**(服务端按(企业,窗口)复用结果,
|
|
15
|
+
// 同参数重复创建返回不同 key 相同数据)——本函数只返回行,去重由 upsert(session_id) 幂等吸收。
|
|
16
|
+
import { callPaced } from './pace.js';
|
|
17
|
+
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
18
|
+
/** 形状 A 载荷里的 message 字符串(create 的任务 key / check 的下载链接都走 message)。 */
|
|
19
|
+
function messageOf(res, endpointId) {
|
|
20
|
+
// 形状 A(envelope:'A'):信封层把 data 放进 payload、原文放 messageText。
|
|
21
|
+
// 实测 create/check 的载荷就是 message 字符串本身(key/链接),不信任何 data 形态。
|
|
22
|
+
const p = res.payload;
|
|
23
|
+
if (typeof p === 'string' && p !== '')
|
|
24
|
+
return p;
|
|
25
|
+
if (typeof res.messageText === 'string' && res.messageText !== '' && res.messageText !== 'OK') {
|
|
26
|
+
return res.messageText;
|
|
27
|
+
}
|
|
28
|
+
throw new Error(`接口 ${endpointId} 未返回可用的 message 载荷(响应形态漂移?)`);
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* 解压密码 zip(ZipCrypto,密码 = appKey 前 12 字符,设计稿 §7.5 实测)并取出 session.txt。
|
|
32
|
+
* fflate 0.8 的 unzipSync 不支持解密 —— 这里手工解析 zip 容器 + ZipCrypto 解密 +
|
|
33
|
+
* inflateRaw(算法经 PKWARE APPNOTE 与真实样本往返验证)。找错密码时校验字节不匹配 → 显式
|
|
34
|
+
* 失败,绝不静默吐乱码。
|
|
35
|
+
* F 低-4:解出的字节用 Node 内置 TextDecoder 解码(不再引 fflate 的 strFromU8 ——
|
|
36
|
+
* 解压/解密路径本来就是零依赖实现,只有这一处解码借了 fflate)。
|
|
37
|
+
* 修复批 4 #6:解压后**用中央目录的 CRC 校验** inflate 结果(crc 非 0 时)——ZipCrypto
|
|
38
|
+
* 的校验字节只有 1 字节、错密码有 1/256 的概率蒙对,CRC 是全量的兜底。
|
|
39
|
+
* ⚠️ crc === 0 时不做比对(写入方未提供 CRC —— 例如某些流式写入器连数据描述符里
|
|
40
|
+
* 也写 0):此时「解压自身失败」仍是兜底(错密码几乎必然让 inflate 报结构错误)。
|
|
41
|
+
* 这条取舍是刻意的:宁可在 crc 缺失时少一层校验,也绝不把 0 当成「真实 CRC」去比对
|
|
42
|
+
* (那会把合法包一律判错)。
|
|
43
|
+
*/
|
|
44
|
+
export async function unzipPasswordZip(zip, appKey) {
|
|
45
|
+
const password = appKey.slice(0, 12);
|
|
46
|
+
const entry = findEntry(zip);
|
|
47
|
+
const raw = zipDecrypt(entry.comp, password, entry);
|
|
48
|
+
const inflated = await inflateRaw(raw);
|
|
49
|
+
if (entry.crc !== 0) {
|
|
50
|
+
const actual = crc32Of(inflated);
|
|
51
|
+
if (actual !== entry.crc) {
|
|
52
|
+
throw new Error(`zip 解压结果 CRC 不符(期望 ${entry.crc.toString(16)},实得 ${actual.toString(16)})` +
|
|
53
|
+
'——密码或数据已损坏,拒绝落库');
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
return new TextDecoder().decode(inflated);
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* 定位名为 session.txt 的条目(导出 zip 内是 session.txt/message.txt 两个 JSONL——
|
|
60
|
+
* 只吃会话,message.txt 即使排在前面也不取;找不到显式抛错)。
|
|
61
|
+
*
|
|
62
|
+
* 修复批 4 #5:**以中央目录为唯一定位来源**。此前逐条目扫本地文件头,遇到 bit3
|
|
63
|
+
*(数据描述符)时本地头 crc/csize 均为 0,步进 `o += 30 + nlen + elen + csize` 会落在
|
|
64
|
+
* 数据描述符上(签名 0x04034b50 之后是 crc/size 字段)→ 循环退出 → **session.txt 永远
|
|
65
|
+
* 找不到**(实测导出 zip 是 message.txt + session.txt 两条目,若写入方用数据描述符即中招;
|
|
66
|
+
* 旧的 bit3 用例只有单条目,掩盖了这条路径)。
|
|
67
|
+
* 现在:EOCD → 中央目录条目(按名匹配)→ 该条目的本地头偏移(CD 偏移 42)+ CD 的
|
|
68
|
+
* csize/crc → 直接算数据起点。数据描述符有无都不影响定位(它夹在数据与下一条目之间,
|
|
69
|
+
* 我们不靠步进遍历)。
|
|
70
|
+
*/
|
|
71
|
+
function findEntry(zip) {
|
|
72
|
+
const u16 = (o) => zip[o] | (zip[o + 1] << 8);
|
|
73
|
+
const u32 = (o) => (zip[o] | (zip[o + 1] << 8) | (zip[o + 2] << 16) | (zip[o + 3] << 24)) >>> 0;
|
|
74
|
+
const cd = centralDirectoryEntry(zip, 'session.txt');
|
|
75
|
+
const local = cd.localOffset;
|
|
76
|
+
if (local + 30 > zip.length || u32(local) !== 0x04034b50) {
|
|
77
|
+
throw new Error('zip 本地文件头偏移不符(中央目录指向的不是本地头)——响应形态漂移,拒绝落库');
|
|
78
|
+
}
|
|
79
|
+
const flags = u16(local + 6);
|
|
80
|
+
const method = u16(local + 8);
|
|
81
|
+
const modTime = u16(local + 10);
|
|
82
|
+
const nlen = u16(local + 26);
|
|
83
|
+
const elen = u16(local + 28);
|
|
84
|
+
if (!(flags & 0x1))
|
|
85
|
+
throw new Error('导出 zip 未加密(与实测形态不符,注册表/文档假设失效)');
|
|
86
|
+
if (method !== 8)
|
|
87
|
+
throw new Error(`不支持的压缩方法:${method}(预期 deflate=8)`);
|
|
88
|
+
const dataStart = local + 30 + nlen + elen;
|
|
89
|
+
if (dataStart + cd.csize > zip.length) {
|
|
90
|
+
throw new Error('zip 声明长度超出文件边界(中央目录与本地头不一致)——拒绝落库');
|
|
91
|
+
}
|
|
92
|
+
return {
|
|
93
|
+
comp: zip.subarray(dataStart, dataStart + cd.csize),
|
|
94
|
+
crc: cd.crc, flags, modTime, method,
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* 中央目录里按名取条目:路径 EOCD(尾部 22 字节定长 + 可选注释,签名 0x06054b50)
|
|
99
|
+
* → CD 起始偏移/条目数 → 逐条 CD 记录(46 字节定长 + 名 + 扩展 + 注释)按名匹配。
|
|
100
|
+
* 返回 CD 里的 crc / csize / 本地头偏移(偏移 42)。找不到/结构不符 → 显式抛错:
|
|
101
|
+
* 宁可失败,也绝不在「无校验字节可对」的状态下把错密码的乱码当数据吐出去。
|
|
102
|
+
* 修复批 4 #5:本函数从「bit3 的兜底补查」升格为**唯一定位来源**(原逐条目本地头
|
|
103
|
+
* 扫描已删除——两步定位必然漂移)。非 bit3 时 CD 的 crc/csize 与本地头一致
|
|
104
|
+
*(APPNOTE 要求),故对本形态无行为变化。
|
|
105
|
+
*/
|
|
106
|
+
function centralDirectoryEntry(zip, name) {
|
|
107
|
+
const u16 = (o) => zip[o] | (zip[o + 1] << 8);
|
|
108
|
+
const u32 = (o) => (zip[o] | (zip[o + 1] << 8) | (zip[o + 2] << 16) | (zip[o + 3] << 24)) >>> 0;
|
|
109
|
+
// EOCD 从尾往前找(签名 0x06054b50;注释最长 64KB)
|
|
110
|
+
let eocd = -1;
|
|
111
|
+
for (let i = zip.length - 22; i >= 0 && i >= zip.length - 22 - 65_535; i--) {
|
|
112
|
+
if (u32(i) === 0x06054b50) {
|
|
113
|
+
eocd = i;
|
|
114
|
+
break;
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
if (eocd < 0)
|
|
118
|
+
throw new Error('zip 缺 central directory 结束记录(EOCD)——无法校验解压密码,拒绝落库');
|
|
119
|
+
const count = u16(eocd + 10);
|
|
120
|
+
let o = u32(eocd + 16);
|
|
121
|
+
const decoder = new TextDecoder();
|
|
122
|
+
for (let i = 0; i < count; i++) {
|
|
123
|
+
if (o + 46 > zip.length || u32(o) !== 0x02014b50) {
|
|
124
|
+
throw new Error('zip central directory 结构不符——无法校验解压密码,拒绝落库');
|
|
125
|
+
}
|
|
126
|
+
const nlen = u16(o + 28);
|
|
127
|
+
const elen = u16(o + 30);
|
|
128
|
+
const clen = u16(o + 32);
|
|
129
|
+
const entryName = decoder.decode(zip.subarray(o + 46, o + 46 + nlen));
|
|
130
|
+
if (entryName === name) {
|
|
131
|
+
return { crc: u32(o + 16), csize: u32(o + 20), localOffset: u32(o + 42) };
|
|
132
|
+
}
|
|
133
|
+
o = o + 46 + nlen + elen + clen;
|
|
134
|
+
}
|
|
135
|
+
throw new Error(`zip central directory 内没有 ${name} 条目——无法校验解压密码,拒绝落库`);
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* ZipCrypto 解密:密钥由密码初始化,逐字节流式更新;校验字节匹配则剥掉 12 字节头。
|
|
139
|
+
* 修复批 4 #6:校验字节的来源取决于 bit3(数据描述符)——
|
|
140
|
+
* · bit3 置位:CRC 被推迟到数据描述符,写入方**无法预先知道** CRC →
|
|
141
|
+
* PKWARE/Info-ZIP 的约定是把加密头第 12 字节写成 **DOS 修改时间的高字节**
|
|
142
|
+
* (`modTime >>> 8`)。此前恒与 CRC 高字节比较,合法的 bit3 包会被判「密码错误」。
|
|
143
|
+
* · bit3 未置位:只有 CRC 高字节一种写法(仍是唯一判据)。
|
|
144
|
+
* 两者都接受(兼容不同写入方):bit3 时 `check === timeHigh || check === crcHigh`。
|
|
145
|
+
* crc === 0(写入方未提供)时不把 0 当真实 CRC 参与比对(否则会把合法包判错)。
|
|
146
|
+
*/
|
|
147
|
+
function zipDecrypt(comp, password, entry) {
|
|
148
|
+
const CRC_TABLE = crcTable();
|
|
149
|
+
const keys = [0x12345678, 0x23456789, 0x34567890];
|
|
150
|
+
const upd = (b) => {
|
|
151
|
+
keys[0] = ((keys[0] >>> 8) ^ CRC_TABLE[(keys[0] ^ b) & 0xFF]) >>> 0;
|
|
152
|
+
keys[1] = (keys[1] + (keys[0] & 0xFF)) >>> 0;
|
|
153
|
+
keys[1] = (Math.imul(keys[1], 134775813) + 1) >>> 0;
|
|
154
|
+
keys[2] = ((keys[2] >>> 8) ^ CRC_TABLE[(keys[2] ^ (keys[1] >>> 24)) & 0xFF]) >>> 0;
|
|
155
|
+
};
|
|
156
|
+
for (const ch of Buffer.from(password, 'utf8'))
|
|
157
|
+
upd(ch);
|
|
158
|
+
const out = new Uint8Array(comp.length);
|
|
159
|
+
for (let i = 0; i < comp.length; i++) {
|
|
160
|
+
const k = ((keys[2] | 2) & 0xFFFF) >>> 0;
|
|
161
|
+
const d = (Math.imul(k, k ^ 1) >>> 8) & 0xFF;
|
|
162
|
+
const p = comp[i] ^ d;
|
|
163
|
+
upd(p);
|
|
164
|
+
out[i] = p;
|
|
165
|
+
}
|
|
166
|
+
// 校验字节 = 加密头最后一字节明文
|
|
167
|
+
const check = out[11];
|
|
168
|
+
const crcHigh = entry.crc === 0 ? null : ((entry.crc >>> 24) & 0xFF);
|
|
169
|
+
const accepted = [];
|
|
170
|
+
if ((entry.flags & 0x8) !== 0)
|
|
171
|
+
accepted.push((entry.modTime >>> 8) & 0xFF); // bit3:修改时间高字节
|
|
172
|
+
if (crcHigh !== null)
|
|
173
|
+
accepted.push(crcHigh);
|
|
174
|
+
if (accepted.length > 0 && !accepted.includes(check)) {
|
|
175
|
+
throw new Error('zip 解压密码错误(校验字节不匹配)——密码应为 appKey 前 12 字符');
|
|
176
|
+
}
|
|
177
|
+
return out.subarray(12);
|
|
178
|
+
}
|
|
179
|
+
/** CRC-32(与 zip 同多项式)。解压结果的全量校验用(修复批 4 #6)。 */
|
|
180
|
+
function crc32Of(buf) {
|
|
181
|
+
const t = crcTable();
|
|
182
|
+
let c = 0xFFFFFFFF;
|
|
183
|
+
for (const b of buf)
|
|
184
|
+
c = (c >>> 8) ^ t[(c ^ b) & 0xFF];
|
|
185
|
+
return (c ^ 0xFFFFFFFF) >>> 0;
|
|
186
|
+
}
|
|
187
|
+
let table = null;
|
|
188
|
+
function crcTable() {
|
|
189
|
+
if (table)
|
|
190
|
+
return table;
|
|
191
|
+
const t = new Uint32Array(256);
|
|
192
|
+
for (let i = 0; i < 256; i++) {
|
|
193
|
+
let c = i;
|
|
194
|
+
for (let k = 0; k < 8; k++)
|
|
195
|
+
c = (c & 1) ? (c >>> 1) ^ 0xEDB88320 : c >>> 1;
|
|
196
|
+
t[i] = c >>> 0;
|
|
197
|
+
}
|
|
198
|
+
table = t;
|
|
199
|
+
return t;
|
|
200
|
+
}
|
|
201
|
+
/** inflateRaw:zip 的 deflate 流是不带 zlib 头的裸 deflate。 */
|
|
202
|
+
async function inflateRaw(data) {
|
|
203
|
+
// DecompressionStream('deflate-raw') 自 Node 20.12 起才被接受(20.10/20.11 抛
|
|
204
|
+
// `format is invalid`,CI 实测确认)——engines 下限即由本能力钉在 >=20.12。
|
|
205
|
+
const ds = new DecompressionStream('deflate-raw');
|
|
206
|
+
const stream = new Blob([data]).stream().pipeThrough(ds);
|
|
207
|
+
const chunks = [];
|
|
208
|
+
const reader = stream.getReader();
|
|
209
|
+
for (;;) {
|
|
210
|
+
const { done, value } = await reader.read();
|
|
211
|
+
if (done)
|
|
212
|
+
break;
|
|
213
|
+
chunks.push(value);
|
|
214
|
+
}
|
|
215
|
+
let len = 0;
|
|
216
|
+
for (const c of chunks)
|
|
217
|
+
len += c.length;
|
|
218
|
+
const out = new Uint8Array(len);
|
|
219
|
+
let o = 0;
|
|
220
|
+
for (const c of chunks) {
|
|
221
|
+
out.set(c, o);
|
|
222
|
+
o += c.length;
|
|
223
|
+
}
|
|
224
|
+
return out;
|
|
225
|
+
}
|
|
226
|
+
/**
|
|
227
|
+
* 导出尾部标记行(C-8):`startTime=…,endTime=…` 的两个毫秒时间戳 = **导出自己的窗口声明**。
|
|
228
|
+
* 实测(private/raw/session.txt,4687 行样本):文件里恰好一行,紧随全部 JSON 行之后,
|
|
229
|
+
* 声明一个 24h 窗口且边界不落在整日/整分上 —— 与「客户端传什么窗口」同形(1 天窗口调用)。
|
|
230
|
+
* 形态判定与解析共用本函数:parseJsonl 的「跳过尾行」与任务层的窗口核对不会各写一套正则。
|
|
231
|
+
* 数值宽度上限 16 位十进制(epoch 毫秒 13 位)同时是**长度上界**:该行最多 ~54 字符,
|
|
232
|
+
* 不存在「无限长原文」——更宽的数字串不是这个形态,按形态漂移显式失败(不静默截断放行)。
|
|
233
|
+
*/
|
|
234
|
+
export function parseExportFixture(line) {
|
|
235
|
+
const m = /^startTime=(\d{1,16}),endTime=(\d{1,16})$/.exec(line);
|
|
236
|
+
return m === null ? null : { start: Number(m[1]), end: Number(m[2]) };
|
|
237
|
+
}
|
|
238
|
+
/** JSONL 解析(Review Focus 4 钉死):逐行 JSON;空行跳过;
|
|
239
|
+
* 末尾非 JSON 尾部标记行(`startTime=...,endTime=...`)跳过并置 tailSkipped=true、
|
|
240
|
+
* 原文带出为 fixture(C-8:该信号此前被调用方 `void` 丢弃,任务层无从核对导出窗口)。
|
|
241
|
+
* 非 JSON 且非尾行 → 显式失败(响应形态漂移必须可见,不能静默丢行)。
|
|
242
|
+
* ⚠️ F 中-1:错误消息**只带行号与长度,不带原文** —— session.txt 的每一行都是 PII
|
|
243
|
+
*(电话/邮箱/地址/IP/CRM 备注),而错误消息会进 stdout、日志与 Agent 上下文;
|
|
244
|
+
* 把行内容截 80 字符贴出去等于把 PII 抄了一份出去(仓库存放区域不可控)。
|
|
245
|
+
* 尾部标记行是唯一的例外(它只有两个整数时间戳、无 PII,且长度由形态限死 ≤54 字符)。
|
|
246
|
+
*/
|
|
247
|
+
export function parseJsonl(text) {
|
|
248
|
+
const rows = [];
|
|
249
|
+
let tailSkipped = false;
|
|
250
|
+
let fixture;
|
|
251
|
+
const lines = text.split('\n');
|
|
252
|
+
for (let i = 0; i < lines.length; i++) {
|
|
253
|
+
const s = lines[i].trim();
|
|
254
|
+
if (s === '')
|
|
255
|
+
continue;
|
|
256
|
+
if (parseExportFixture(s) !== null) {
|
|
257
|
+
tailSkipped = true;
|
|
258
|
+
fixture = s;
|
|
259
|
+
continue;
|
|
260
|
+
}
|
|
261
|
+
try {
|
|
262
|
+
rows.push(JSON.parse(s));
|
|
263
|
+
}
|
|
264
|
+
catch {
|
|
265
|
+
// 不回显原文(PII 红线):行号 + 字节长度足以定位与判断形态漂移。
|
|
266
|
+
throw new Error(`session.txt 第 ${i + 1} 行无法解析为 JSON(且不是已知的尾部标记行):` +
|
|
267
|
+
`该行长度 ${s.length} 字节。原始行内容不回显(含用户 PII),请直接查看导出文件对应行`);
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
return { rows, tailSkipped, ...(fixture === undefined ? {} : { fixture }) };
|
|
271
|
+
}
|
|
272
|
+
/**
|
|
273
|
+
* 四步流水线:create → poll → download(ctx.fetchZip) → unzip + parse。
|
|
274
|
+
* 返回的 sessions 是未 enrich 的原始行(列投影/派生在 sessions.ts 的任务层);
|
|
275
|
+
* `fixture` 是导出尾部标记行的原文(C-8:**不再 `void` 丢弃** —— 它是导出自己声明的
|
|
276
|
+
* 窗口,交由调用方与请求窗口核对;缺失/不符都在任务层有出口)。
|
|
277
|
+
*/
|
|
278
|
+
export async function runExportPipeline(w, ctx) {
|
|
279
|
+
if (!Number.isFinite(w.start) || !Number.isFinite(w.end) || w.end <= w.start) {
|
|
280
|
+
throw usage(`会话导出窗口无效:start=${w.start}, end=${w.end}(start==end / start>end / 超限同为 14004,无法区分——必须本地保证有效)`);
|
|
281
|
+
}
|
|
282
|
+
// —— 1. create:{start, end}(毫秒;code 实测可选且不校验,不传误导性参数)——
|
|
283
|
+
// B 高-1:走 callPaced —— 裸 call 让本地令牌桶(export.session 10 秒 6 次)的拒绝
|
|
284
|
+
// 直接冒泡成 fetch_failed gap,远端限频的证据(冷却行)也没人写。
|
|
285
|
+
const created = await callPaced('export.session', { start: w.start, end: w.end }, ctx);
|
|
286
|
+
const key = messageOf(created, 'export.session');
|
|
287
|
+
// —— 2. poll:14403(core 判 kind=pending 抛出)重试,有界 ——
|
|
288
|
+
const maxPolls = ctx.maxPolls ?? 120;
|
|
289
|
+
let link = '';
|
|
290
|
+
for (let i = 0; i < maxPolls; i++) {
|
|
291
|
+
if (ctx.pollIntervalMs !== undefined ? ctx.pollIntervalMs > 0 : true) {
|
|
292
|
+
await sleep(ctx.pollIntervalMs ?? 5_000);
|
|
293
|
+
}
|
|
294
|
+
try {
|
|
295
|
+
// export.session.check 在限频表里显式登记为 null(无限频)—— 仍走 callPaced
|
|
296
|
+
// 以统一路径(本地桶/错误形态处理一致;闸门侧 null 登记即豁免,不写冷却行)。
|
|
297
|
+
const checked = await callPaced('export.session.check', { key }, ctx);
|
|
298
|
+
link = messageOf(checked, 'export.session.check');
|
|
299
|
+
break;
|
|
300
|
+
}
|
|
301
|
+
catch (e) {
|
|
302
|
+
if (e.code === '14403')
|
|
303
|
+
continue;
|
|
304
|
+
throw e;
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
if (link === '') {
|
|
308
|
+
throw new Error(`会话导出轮询超限:连续 ${maxPolls} 次 14403(任务仍未完成)——key=${key.slice(0, 8)}…`);
|
|
309
|
+
}
|
|
310
|
+
// —— 3/4. download + unzip + parse ——
|
|
311
|
+
const zip = await ctx.fetchZip(link);
|
|
312
|
+
const text = await unzipPasswordZip(zip, ctx.credentials.appKey);
|
|
313
|
+
const { rows, fixture } = parseJsonl(text);
|
|
314
|
+
return { sessions: rows, ...(fixture === undefined ? {} : { fixture }) };
|
|
315
|
+
}
|
|
316
|
+
/** usage 错误(退出码 2):本地窗口参数问题,重试无用。 */
|
|
317
|
+
function usage(msg) {
|
|
318
|
+
const e = new Error(msg);
|
|
319
|
+
e.kind = 'usage';
|
|
320
|
+
return e;
|
|
321
|
+
}
|