qiyu-cli 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +464 -2
- package/dist/cli.js +248 -0
- package/dist/commands/api.js +770 -0
- package/dist/commands/auth.js +349 -0
- package/dist/commands/credentials-source.js +70 -0
- package/dist/commands/db.js +503 -0
- package/dist/commands/local-fallback-wiring.js +67 -0
- package/dist/commands/local-fallback.js +34 -0
- package/dist/commands/query.js +564 -0
- package/dist/commands/skill-specs.js +64 -0
- package/dist/commands/skill.js +863 -0
- package/dist/commands/spec-types.js +145 -0
- package/dist/commands/stats.js +299 -0
- package/dist/commands/store-flags.js +56 -0
- package/dist/commands/store.js +258 -0
- package/dist/commands/sync-run.js +479 -0
- package/dist/commands/sync.js +757 -0
- package/dist/commands/tree.js +27 -0
- package/dist/core/config.js +259 -0
- package/dist/core/credentials.js +71 -0
- package/dist/core/edition.js +18 -0
- package/dist/core/envelope.js +201 -0
- package/dist/core/errors.js +356 -0
- package/dist/core/http.js +348 -0
- package/dist/core/ratelimit.js +126 -0
- package/dist/core/registry/index.js +212 -0
- package/dist/core/registry/params/ai.knowledge.addKnowledge.json +88 -0
- package/dist/core/registry/params/ai.knowledge.deleteKnowledges.json +46 -0
- package/dist/core/registry/params/ai.knowledge.editChannelAnswer.json +60 -0
- package/dist/core/registry/params/ai.knowledge.editKnowledge.json +95 -0
- package/dist/core/registry/params/ai.knowledge.editKnowledgeNoSimilars.json +102 -0
- package/dist/core/registry/params/ai.knowledge.getCategoryTree.json +39 -0
- package/dist/core/registry/params/ai.knowledge.getDetail.json +46 -0
- package/dist/core/registry/params/ai.knowledge.search.json +67 -0
- package/dist/core/registry/params/ai.statistic.transferHuman.json +60 -0
- package/dist/core/registry/params/category.list.json +39 -0
- package/dist/core/registry/params/chat.queue.group.query.json +39 -0
- package/dist/core/registry/params/crm.modifyOrder.json +48 -0
- package/dist/core/registry/params/crm.modifyUser.json +48 -0
- package/dist/core/registry/params/crm.syncCrmInfo.json +46 -0
- package/dist/core/registry/params/crm.tagChange.json +74 -0
- package/dist/core/registry/params/data.overview.ipcc.json +32 -0
- package/dist/core/registry/params/data.overview.session.json +53 -0
- package/dist/core/registry/params/evaluation.setting.get.json +39 -0
- package/dist/core/registry/params/event.applyStaff.json +116 -0
- package/dist/core/registry/params/event.evaluate.json +74 -0
- package/dist/core/registry/params/event.inviteEvaluate.json +81 -0
- package/dist/core/registry/params/event.qcResult.json +166 -0
- package/dist/core/registry/params/event.queryQueueStatus.json +39 -0
- package/dist/core/registry/params/event.queueEnter.json +40 -0
- package/dist/core/registry/params/event.queueLeave.json +46 -0
- package/dist/core/registry/params/event.quitQueue.json +39 -0
- package/dist/core/registry/params/event.sessionChange.json +95 -0
- package/dist/core/registry/params/event.sessionData.json +452 -0
- package/dist/core/registry/params/event.sessionEnd.json +117 -0
- package/dist/core/registry/params/event.sessionStart.json +109 -0
- package/dist/core/registry/params/event.updateUInfo.json +46 -0
- package/dist/core/registry/params/export.session.check.json +7 -0
- package/dist/core/registry/params/export.session.json +7 -0
- package/dist/core/registry/params/export.session.one.json +7 -0
- package/dist/core/registry/params/export.session.one.message.json +7 -0
- package/dist/core/registry/params/get_call_order.json +50 -0
- package/dist/core/registry/params/get_call_user_info.json +36 -0
- package/dist/core/registry/params/get_order.json +50 -0
- package/dist/core/registry/params/get_token.json +22 -0
- package/dist/core/registry/params/get_user_info.json +36 -0
- package/dist/core/registry/params/icoutcall.callIntercept.list.json +32 -0
- package/dist/core/registry/params/icoutcall.dialogFlow.fuzzySearch.json +53 -0
- package/dist/core/registry/params/icoutcall.dialogFlow.getDialogFlowList.json +32 -0
- package/dist/core/registry/params/icoutcall.get.customer.json +39 -0
- package/dist/core/registry/params/icoutcall.get.jobdetail.json +39 -0
- package/dist/core/registry/params/icoutcall.get.jobproperties.json +39 -0
- package/dist/core/registry/params/icoutcall.history.seat.json +67 -0
- package/dist/core/registry/params/icoutcall.history.team.json +67 -0
- package/dist/core/registry/params/icoutcall.job.create.json +53 -0
- package/dist/core/registry/params/icoutcall.job.importcustomer.json +46 -0
- package/dist/core/registry/params/icoutcall.job.pause.json +39 -0
- package/dist/core/registry/params/icoutcall.job.readdcustomer.json +109 -0
- package/dist/core/registry/params/icoutcall.job.start.json +39 -0
- package/dist/core/registry/params/icoutcall.job.terminate.json +39 -0
- package/dist/core/registry/params/icoutcall.phone.getLineList.json +32 -0
- package/dist/core/registry/params/ipcc.calltask.add.auto.json +117 -0
- package/dist/core/registry/params/ipcc.calltask.add.byCustom.json +96 -0
- package/dist/core/registry/params/ipcc.calltask.add.byRelation.json +82 -0
- package/dist/core/registry/params/ipcc.calltask.add.toGroup.json +89 -0
- package/dist/core/registry/params/ipcc.calltask.add.toStaff.json +89 -0
- package/dist/core/registry/params/ipcc.calltask.get.json +47 -0
- package/dist/core/registry/params/ipcc.calltaskdetail.add.json +54 -0
- package/dist/core/registry/params/ipcc.calltaskdetail.delete.json +40 -0
- package/dist/core/registry/params/ipcc.calltaskdetail.get.json +54 -0
- package/dist/core/registry/params/ipcc.calltaskdetail.update.json +40 -0
- package/dist/core/registry/params/ipcc.customfield.template.get.json +47 -0
- package/dist/core/registry/params/ipcc.login.json +47 -0
- package/dist/core/registry/params/kefu.add.json +130 -0
- package/dist/core/registry/params/kefu.delete.json +39 -0
- package/dist/core/registry/params/kefu.update.json +137 -0
- package/dist/core/registry/params/kefu.v2.query.json +39 -0
- package/dist/core/registry/params/kefuGroup.add.json +39 -0
- package/dist/core/registry/params/kefuGroup.delete.json +39 -0
- package/dist/core/registry/params/kefuGroup.update.json +46 -0
- package/dist/core/registry/params/kefuGroup.updateKefus.json +46 -0
- package/dist/core/registry/params/merchant.account.login.json +53 -0
- package/dist/core/registry/params/merchant.changePassword.json +46 -0
- package/dist/core/registry/params/merchant.create.json +81 -0
- package/dist/core/registry/params/merchant.disable.json +39 -0
- package/dist/core/registry/params/merchant.enable.json +39 -0
- package/dist/core/registry/params/merchant.list.json +46 -0
- package/dist/core/registry/params/merchant.query.json +46 -0
- package/dist/core/registry/params/merchant.update.json +74 -0
- package/dist/core/registry/params/message.send.image.json +53 -0
- package/dist/core/registry/params/message.send.json +53 -0
- package/dist/core/registry/params/message.send.voice.json +53 -0
- package/dist/core/registry/params/message.sendFile.json +39 -0
- package/dist/core/registry/params/message.uploadFile.json +39 -0
- package/dist/core/registry/params/online.session.category.list.json +39 -0
- package/dist/core/registry/params/online.session.customfield.template.json +39 -0
- package/dist/core/registry/params/online.session.customfield.update.json +74 -0
- package/dist/core/registry/params/order.upload.json +40 -0
- package/dist/core/registry/params/push.msg.image.json +74 -0
- package/dist/core/registry/params/push.msg.richText.json +74 -0
- package/dist/core/registry/params/push.msg.text.json +74 -0
- package/dist/core/registry/params/receive.message.json +89 -0
- package/dist/core/registry/params/robot.data.getQuestionsByCategoryName.json +67 -0
- package/dist/core/registry/params/robot.data.knowledge.json +46 -0
- package/dist/core/registry/params/robot.data.kpoint.json +46 -0
- package/dist/core/registry/params/robot.data.queryStandardByEffectiveQuestion.json +46 -0
- package/dist/core/registry/params/robot.data.queryStandardQuestionList.json +46 -0
- package/dist/core/registry/params/robot.evaluation.comment.json +67 -0
- package/dist/core/registry/params/robot.evaluation.setting.json +46 -0
- package/dist/core/registry/params/robot.session.assist.json +74 -0
- package/dist/core/registry/params/robot.session.chat.json +74 -0
- package/dist/core/registry/params/robot.session.close.json +53 -0
- package/dist/core/registry/params/robot.session.commonQuestions.json +74 -0
- package/dist/core/registry/params/robot.session.create.json +81 -0
- package/dist/core/registry/params/robot.session.evaluation.json +67 -0
- package/dist/core/registry/params/robot.session.msgAssociate.json +46 -0
- package/dist/core/registry/params/robot.session.msgEvaluation.json +53 -0
- package/dist/core/registry/params/robot.session.msgPredict.json +74 -0
- package/dist/core/registry/params/robot.session.transfer.json +36 -0
- package/dist/core/registry/params/robot.session.uploadFile.json +32 -0
- package/dist/core/registry/params/robot.statistic.summary.json +53 -0
- package/dist/core/registry/params/robot.transfer.config.json +15 -0
- package/dist/core/registry/params/session.user.profile.json +39 -0
- package/dist/core/registry/params/settings.wxTemplates.json +46 -0
- package/dist/core/registry/params/smstask.create.json +53 -0
- package/dist/core/registry/params/smstask.status.json +39 -0
- package/dist/core/registry/params/staff.group.list.new.json +39 -0
- package/dist/core/registry/params/staff.group.members.new.json +46 -0
- package/dist/core/registry/params/staff.list.new.json +46 -0
- package/dist/core/registry/params/staff.login.json +47 -0
- package/dist/core/registry/params/statistic.overview.json +61 -0
- package/dist/core/registry/params/statistic.performance.json +82 -0
- package/dist/core/registry/params/statistic.salaryDetail.json +54 -0
- package/dist/core/registry/params/statistic.satisfaction.report.json +68 -0
- package/dist/core/registry/params/statistic.staffAttendance.json +75 -0
- package/dist/core/registry/params/statistic.staffquality.json +75 -0
- package/dist/core/registry/params/statistic.staffworklod.json +68 -0
- package/dist/core/registry/params/tag.all.json +32 -0
- package/dist/core/registry/params/tag.batchUpdate.json +32 -0
- package/dist/core/registry/params/tag.update.json +60 -0
- package/dist/core/registry/params/ticket.apply.json +46 -0
- package/dist/core/registry/params/ticket.batch.modify.json +53 -0
- package/dist/core/registry/params/ticket.create.json +151 -0
- package/dist/core/registry/params/ticket.filter.count.json +46 -0
- package/dist/core/registry/params/ticket.filter.list.json +46 -0
- package/dist/core/registry/params/ticket.finish.json +60 -0
- package/dist/core/registry/params/ticket.followers.update.json +60 -0
- package/dist/core/registry/params/ticket.list.json +74 -0
- package/dist/core/registry/params/ticket.log.json +39 -0
- package/dist/core/registry/params/ticket.modify.json +74 -0
- package/dist/core/registry/params/ticket.new.detail.json +39 -0
- package/dist/core/registry/params/ticket.reminder.json +60 -0
- package/dist/core/registry/params/ticket.reopen.json +67 -0
- package/dist/core/registry/params/ticket.reply.json +60 -0
- package/dist/core/registry/params/ticket.search.json +116 -0
- package/dist/core/registry/params/ticket.template.fields.json +39 -0
- package/dist/core/registry/params/ticket.template.list.json +39 -0
- package/dist/core/registry/params/ticket.transfer.json +74 -0
- package/dist/core/registry/params/track.event.set.json +32 -0
- package/dist/core/registry/params/wecom.robot.switch.json +43 -0
- package/dist/core/registry/params.js +84 -0
- package/dist/core/registry/specs.json +2172 -0
- package/dist/core/registry/validate.json +11 -0
- package/dist/core/registry/validation-overrides.js +98 -0
- package/dist/core/sanitize.js +24 -0
- package/dist/core/sign.js +21 -0
- package/dist/core/time.js +194 -0
- package/dist/core/validate.js +140 -0
- package/dist/output.js +263 -0
- package/dist/postinstall.js +178 -0
- package/dist/query/errors.js +395 -0
- package/dist/query/mask.js +368 -0
- package/dist/query/metric-catalog.js +85 -0
- package/dist/query/sandbox.js +544 -0
- package/dist/query/table-query.js +297 -0
- package/dist/query/time-shape.js +44 -0
- package/dist/schedule/install.js +377 -0
- package/dist/skill/check.js +429 -0
- package/dist/skill/detect.js +336 -0
- package/dist/skill/install.js +1202 -0
- package/dist/skill/paths.js +343 -0
- package/dist/skill/qiyu/SKILL.md +86 -0
- package/dist/skill/qiyu/errors.md +55 -0
- package/dist/skill/qiyu/forbidden.md +52 -0
- package/dist/skill/qiyu/metrics.md +65 -0
- package/dist/skill/qiyu/protocols.md +55 -0
- package/dist/skill/qiyu/setup.md +50 -0
- package/dist/skill/refs.js +379 -0
- package/dist/store/config.js +137 -0
- package/dist/store/duckdb.js +321 -0
- package/dist/store/index.js +18 -0
- package/dist/store/retry.js +36 -0
- package/dist/store/schema.js +419 -0
- package/dist/store/types.js +4 -0
- package/dist/sync/day.js +8 -0
- package/dist/sync/engine.js +818 -0
- package/dist/sync/rate-gate.js +95 -0
- package/dist/sync/registry.js +22 -0
- package/dist/sync/state.js +114 -0
- package/dist/sync/tasks/dim.js +186 -0
- package/dist/sync/tasks/knowledge.js +252 -0
- package/dist/sync/tasks/pace.js +66 -0
- package/dist/sync/tasks/pipeline.js +321 -0
- package/dist/sync/tasks/reports.js +448 -0
- package/dist/sync/tasks/sessions.js +188 -0
- package/dist/sync/tasks/tickets.js +219 -0
- package/dist/sync/types.js +1 -0
- package/dist/sync/windowing.js +56 -0
- package/package.json +52 -4
|
@@ -0,0 +1,818 @@
|
|
|
1
|
+
import { readWatermark, markPending, advanceWatermark, recordGap, clearGap, clearLock, } from './state.js';
|
|
2
|
+
import { sliceByMaxWindow } from './windowing.js';
|
|
3
|
+
import { isLockError } from '../store/retry.js';
|
|
4
|
+
import { clampGateWait } from './rate-gate.js';
|
|
5
|
+
import { shanghaiDayIso, shanghaiDayStart, shanghaiYesterdayStart } from './day.js';
|
|
6
|
+
const DAY = 86_400_000;
|
|
7
|
+
/** 回看毫秒数:水位续跑的起点回退量(overlapDays 天;0 = 不回看)。 */
|
|
8
|
+
const overlapDaysOf = (task) => (task.overlapDays ?? 0) * DAY;
|
|
9
|
+
/** 盖在 sync_state.locked_by 上的进程标识(E 中-1:status 的 locked 字段据此为真)。 */
|
|
10
|
+
const lockOwner = () => `pid ${process.pid}`;
|
|
11
|
+
/** 引擎自己的「现在」:ctx.now 优先(命令层给任务层固定 clock 时引擎仍可用真实时刻)。 */
|
|
12
|
+
const nowOf = (ctx) => (ctx.now ?? ctx.clock)();
|
|
13
|
+
/** 限频闸门的键(B 中-6):endpoint id 优先,缺省回退任务 id。 */
|
|
14
|
+
const gateKeyOf = (task) => task.endpoint ?? task.id;
|
|
15
|
+
/**
|
|
16
|
+
* 首跑起点的合理下界(C 中-2):起点是 0(或负数)时它是「从头开始」的**哨兵**,
|
|
17
|
+
* 不是真实日期 —— 1970 年没有任何数据,而 `ticket.search` 的 `opStart` 还会被服务端
|
|
18
|
+
* **静默钳到「now − 31 天」**(发 start=0 等于把窗口的解释权交给服务端,客户端以为
|
|
19
|
+
* 「从最早开始」,实际拿到的是被改写过的窗口)。任务声明 `windowStartFloorDays` 后,
|
|
20
|
+
* 哨兵起点被抬到 `now − N 天`,抬升结果如实回显在 resolved_range 里(不静默)。
|
|
21
|
+
* 显式给**正数**起点(--since/--until)则原样照发,绝不改写调用方的区间。
|
|
22
|
+
* ⚠️ 请求的 end 早于下界时(如 `--until` 指向 40 天前)**不抬升** —— 那段历史该接口
|
|
23
|
+
* 物理上给不出,抬升会把「请求了拿不到的数据」静默变成「0 窗 complete」;保持哨兵起点
|
|
24
|
+
* 让任务层守卫显式失败(落 gap、可见),绝不静默成功。
|
|
25
|
+
*/
|
|
26
|
+
function floorRangeStart(task, rangeStart, rangeEnd, nowMs) {
|
|
27
|
+
if (task.windowStartFloorDays === undefined || rangeStart > 0)
|
|
28
|
+
return rangeStart;
|
|
29
|
+
const floor = nowMs - task.windowStartFloorDays * DAY;
|
|
30
|
+
return rangeEnd > floor ? floor : rangeStart;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* 窗口任务的起点合成(runWindowed 与 planRotation 共用,B 中-5/C 中-2 的统一落点):
|
|
34
|
+
* ① 显式区间(--full/--since/--until)优先 —— 水位**不参与**抬升(--since 是「把这段
|
|
35
|
+
* 历史回补一遍」的指令,被水位抬升会静默缩成一窗且输出仍是 complete);
|
|
36
|
+
* ② 缺省路径走水位续跑:watermark+1ms − overlapDays 回看(T9 裁定);
|
|
37
|
+
* ③ **起点 ≤ 0 是哨兵**(--full 的 start=0、仅 --until 的 start=0、首跑水位为空)→
|
|
38
|
+
* 声明了 windowStartFloorDays 的任务抬到 now − N 天(C 中-2)。该下界属于任务声明
|
|
39
|
+
* (接口物理上给不出更早的数据),不是对调用方区间的改写;显式正数起点不受影响。
|
|
40
|
+
*/
|
|
41
|
+
function resolveRangeStart(task, ctx, state, requested, nowMs) {
|
|
42
|
+
let start = requested.start;
|
|
43
|
+
if (ctx.rangeOverride === undefined && state.watermark !== null) {
|
|
44
|
+
const wm = Date.parse(state.watermark);
|
|
45
|
+
if (!Number.isNaN(wm)) {
|
|
46
|
+
// T9 裁定接线:overlapDays 是「回看重叠」——水位续跑的窗口起点 = watermark+1 − overlap。
|
|
47
|
+
// 工单按操作时间增量时,3 天回看覆盖乱序到达的状态变更/日志(upsert 幂等吸收重叠行)。
|
|
48
|
+
const raw = wm + 1 - overlapDaysOf(task);
|
|
49
|
+
// 修复批 5 #1(日对齐任务):续跑起点落到**上海日界**(向下取整到当日 0:00)——
|
|
50
|
+
// 水位所在那一天整体重拉,写出的 stat_date 行才是全天聚合。若按 raw(水位+1ms)
|
|
51
|
+
// 起,首窗是 [水位+1ms, 次日0点) 的半天切片,会把该日的全天行覆盖成半天聚合
|
|
52
|
+
// (「每日概览」语义每天漂移的根因)。显式区间(--since/--until/--full)不走此
|
|
53
|
+
// 分支:调用方给的起点原样照发(B 中-5,见 floorRangeStart)。
|
|
54
|
+
start = Math.max(start, task.windowAlign === 'shanghaiDay' ? shanghaiDayStart(raw) : raw);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
return floorRangeStart(task, start, requested.end, nowMs);
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* gap detail 的中文包装(C 低-2):gap 要进 stdout 与持久层(sync_gaps),
|
|
61
|
+
* 而失败原因可能是任意来源 —— `fetch failed`、DuckDB 的英文原文、上游报文片段。
|
|
62
|
+
* 沿 store 层 thrownToExit 的形态:中文前缀说明「这是什么失败」,底层原文照附。
|
|
63
|
+
* ⚠️ 引擎自己抛的中文结论(配对核对/本地限频)不包装 —— 再套一层只会变噪音。
|
|
64
|
+
*/
|
|
65
|
+
function failDetail(label, e) {
|
|
66
|
+
const raw = e instanceof Error ? e.message : String(e);
|
|
67
|
+
return `${label}(底层:${raw})`;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* 写策略落地(事务内调用):deleteRangeInsert 按范围列删旧 + upsert;upsertRows 直接 upsert。
|
|
71
|
+
* ⚠️ 窗口是**半开** [start, end),而删除条件是闭区间 —— 天对齐分支的上界必须取
|
|
72
|
+
* `dayIso(end − 1ms)`(窗口内最后一个瞬时所在的自然日),否则 end 恰落在日界
|
|
73
|
+
*(`--until` 指定日期 / windowEndClamp 钳到昨天 0:00 / 日对齐水位都会产生)时
|
|
74
|
+
* `dayIso(end)` = 下一天零点,删除会多删**下一天**的行:`--since D0 --until D2`
|
|
75
|
+
* 的第二窗 end=D2 零点 → 删除范围含 D2 → D2 日已同步的行被删而本轮不重拉(D2 在
|
|
76
|
+
* 请求区间外,永久损失)。`end − 1ms` 后:1 天窗 [D, D+1) → 删除恰好 [D, D],
|
|
77
|
+
* 31 天考勤窗 [D0, D0+31d) → [D0, D0+30](= 31 天),与写入行集一致。
|
|
78
|
+
*/
|
|
79
|
+
async function writeRows(task, ops, rows, range) {
|
|
80
|
+
if (task.write.kind === 'deleteRangeInsert') {
|
|
81
|
+
const dayAligned = task.write.rangeAlign === 'shanghaiDay';
|
|
82
|
+
const from = dayAligned ? shanghaiDayIso(range.start) : new Date(range.start).toISOString();
|
|
83
|
+
const to = dayAligned ? shanghaiDayIso(range.end - 1) : new Date(range.end).toISOString();
|
|
84
|
+
// where 是可信代码常量(同 query 红线):列名来自任务声明(代码级),值参数化绑定。
|
|
85
|
+
// 范围边界与行内日期列同口径(rangeAlign):行写「上海天对齐的日期列」时,删除范围
|
|
86
|
+
// 也必须是天对齐边界——否则 --full 与增量对同一业务日的删除范围互不覆盖(A 中-2)。
|
|
87
|
+
for (const col of task.write.rangeColumns) {
|
|
88
|
+
await ops.delete(task.table, `"${col}" >= ? AND "${col}" <= ?`, [from, to]);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
if (rows.length === 0)
|
|
92
|
+
return 0;
|
|
93
|
+
return ops.upsert(task.table, rows);
|
|
94
|
+
}
|
|
95
|
+
/** enrich 过滤:返回 kept(应写行)与 dropped(正当淘汰行数)。 */
|
|
96
|
+
function applyEnrich(task, rows) {
|
|
97
|
+
if (!task.enrich)
|
|
98
|
+
return { kept: rows, dropped: 0 };
|
|
99
|
+
const kept = [];
|
|
100
|
+
let dropped = 0;
|
|
101
|
+
for (const row of rows) {
|
|
102
|
+
const out = task.enrich(row);
|
|
103
|
+
if (out === null) {
|
|
104
|
+
dropped++;
|
|
105
|
+
continue;
|
|
106
|
+
}
|
|
107
|
+
kept.push(out);
|
|
108
|
+
}
|
|
109
|
+
return { kept, dropped };
|
|
110
|
+
}
|
|
111
|
+
/** 引擎层等待(中-2①):默认真实 setTimeout;ctx.sleep 可注入(测试假时钟)。 */
|
|
112
|
+
const engineSleep = (ctx, ms) => ctx.sleep !== undefined ? ctx.sleep(ms) : new Promise((r) => setTimeout(r, ms));
|
|
113
|
+
/**
|
|
114
|
+
* 有界重试(B 中-3 / B 中-4):固定间隔、**次数封顶**。次数封顶而非耗时封顶是刻意的 ——
|
|
115
|
+
* 注入时钟可能是冻结的(确定性测试),耗时预算在冻结时钟下永不终止。
|
|
116
|
+
* 首次失败后重试 RETRY_ATTEMPTS 次,仍失败则抛错(调用方记 gap / 上抛)。
|
|
117
|
+
* ⚠️ 只重试「值得重试」的失败(isTransient):本地参数/形状错误重试必然同结论,
|
|
118
|
+
* 白花两次往返与两段等待。
|
|
119
|
+
*/
|
|
120
|
+
const RETRY_ATTEMPTS = 2;
|
|
121
|
+
const RETRY_DELAY_MS = 500;
|
|
122
|
+
async function withRetries(ctx, op, retryable = () => true) {
|
|
123
|
+
let last;
|
|
124
|
+
for (let attempt = 0;; attempt++) {
|
|
125
|
+
try {
|
|
126
|
+
return await op();
|
|
127
|
+
}
|
|
128
|
+
catch (e) {
|
|
129
|
+
last = e;
|
|
130
|
+
if (attempt >= RETRY_ATTEMPTS || !retryable(e))
|
|
131
|
+
break;
|
|
132
|
+
await engineSleep(ctx, RETRY_DELAY_MS);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
throw last;
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* 瞬时失败判据(B 中-3/B 中-4):以错误自带的 retryable 标记为准。
|
|
139
|
+
* ⚠️ 重试预算的**归属边界**(刻意收窄,避免把等待乘起来):
|
|
140
|
+
* · 网络失败(kind=network)**不在此列** —— core `http.call` 已自带 2 次退避重试
|
|
141
|
+
* (实测 300ms/600ms),引擎再套一层等于 3 倍往返与 3 倍等待,把 core 的预算语义
|
|
142
|
+
* 变成噪声(实测形态:一次「fetch failed」会从 3 次请求变成 9 次);
|
|
143
|
+
* · 限频(kind=rate_limit)不在此列 —— 任务层 callPaced 已有自己的等待预算,
|
|
144
|
+
* 引擎层再等一轮只会把冷却乘起来(限频靠冷却解决,不靠重试);
|
|
145
|
+
* · 本地参数错误(usage)、配对核对结论(count_mismatch)、任务层自抛的形态漂移
|
|
146
|
+
* 都没有 retryable 标记 → 不重试(重试必然同结论)。
|
|
147
|
+
* 于是引擎重试真正覆盖的是 core 无法归类/无法重试的那一类:撞锁(store_locked)
|
|
148
|
+
* 与任务层显式标注 retryable 的瞬时错误。
|
|
149
|
+
*/
|
|
150
|
+
function isTransient(e) {
|
|
151
|
+
const err = e;
|
|
152
|
+
if (err.kind === 'store_locked')
|
|
153
|
+
return true;
|
|
154
|
+
if (err.kind === 'rate_limit' || err.kind === 'network')
|
|
155
|
+
return false;
|
|
156
|
+
return err.retryable === true;
|
|
157
|
+
}
|
|
158
|
+
/** store 事务的撞锁重试(B 中-4,spec §6.3:C1 写者无内置重试,重试调度归 D 层)。 */
|
|
159
|
+
const storeRetryable = (e) => e.kind === 'store_locked' || isLockError(e);
|
|
160
|
+
const withStoreRetries = (ctx, op) => withRetries(ctx, op, storeRetryable);
|
|
161
|
+
/** 本窗 gap 落库 + 收集(独立事务:本窗已回滚,gap 不能混进失败事务)。 */
|
|
162
|
+
async function storeGap(ctx, task, w, reason, detail, gaps) {
|
|
163
|
+
const info = { rangeStart: String(w.start), rangeEnd: String(w.end), reason, detail };
|
|
164
|
+
await withStoreRetries(ctx, () => ctx.store.transaction(async (tx) => {
|
|
165
|
+
await recordGap(tx, task.id, info.rangeStart, info.rangeEnd, reason, detail, { now: ctx.clock });
|
|
166
|
+
}));
|
|
167
|
+
gaps.push(info);
|
|
168
|
+
}
|
|
169
|
+
/**
|
|
170
|
+
* gapHint → sync_gaps.detail(C-8):hint 带 `fixture`(会话导出的尾部标记行原文)时
|
|
171
|
+
* 追加标注 —— 「为什么可疑」在 detail 正文、「可疑的那行原文」在其后,同一行可见。
|
|
172
|
+
* 不带 fixture 的 hint(dim 的 staffIdList 形态漂移等)detail 原样落库,行为不变。
|
|
173
|
+
*/
|
|
174
|
+
function gapHintDetail(hint) {
|
|
175
|
+
return hint.fixture === undefined ? hint.detail : `${hint.detail}(fixture:${hint.fixture})`;
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* 限频闸门获取(中-2①「慢不是错」):冷却中不再立即记 gap 放弃本窗 ——
|
|
179
|
+
* 按 retryAfter 等待(注入 sleep 可测)后重试 acquire,最多 GATE_WAIT_BUDGET 次;
|
|
180
|
+
* 预算耗尽才抛出(调用方记 rate_limit_local gap)。等待是纯时钟推进,不碰网络,
|
|
181
|
+
* 不会超发;预算只封顶「单窗在 gate 上空转的次数」,冷却行坏数据时有限步终止。
|
|
182
|
+
* 等待时长本身也有上限(B 低-2):冷却行里算出天文数字时按 MAX_GATE_WAIT_MS 封顶,
|
|
183
|
+
* 剩下的交给等待预算终止 —— 宁可多等几轮,也绝不无限等。
|
|
184
|
+
*/
|
|
185
|
+
const GATE_WAIT_BUDGET = 3;
|
|
186
|
+
const GATE_WAIT_FALLBACK_MS = 1_000;
|
|
187
|
+
async function acquireGated(ctx, task) {
|
|
188
|
+
const key = gateKeyOf(task);
|
|
189
|
+
for (let waits = 0;; waits++) {
|
|
190
|
+
try {
|
|
191
|
+
// 冷却是持久化的:读库也在 store 上,撞锁走同一有界重试(B 中-4)。
|
|
192
|
+
await withStoreRetries(ctx, () => ctx.gate.acquire(key));
|
|
193
|
+
return;
|
|
194
|
+
}
|
|
195
|
+
catch (e) {
|
|
196
|
+
if (e.kind !== 'rate_limit_local' || waits >= GATE_WAIT_BUDGET)
|
|
197
|
+
throw e;
|
|
198
|
+
const wait = clampGateWait(e.retryAfter ?? 0) || GATE_WAIT_FALLBACK_MS;
|
|
199
|
+
await engineSleep(ctx, wait);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
/**
|
|
204
|
+
* 单窗原子提交(红线 3):写数据 + 推水位 + 清缺口。配对不符在此抛出 → 整体回滚。
|
|
205
|
+
* watermarkRange(中-2② + 修复批 4 #2):事务内 advanceWatermark 的**推进值**(水位)
|
|
206
|
+
* 与**覆盖区间**(resolved_range)的唯一来源 —— 未断链 = 本窗 end;断链后冻结在
|
|
207
|
+
* 连续成功末端(水位绝不越过失败窗)。两者必须同源,否则 `sync status` 会谎报覆盖。
|
|
208
|
+
* null = 断链且本轮尚无任何成功窗 → 不推水位(保持 markPending 的既有状态),
|
|
209
|
+
* 数据与清缺口照常提交 —— 绝不伪造推进点。红线 3 不变:推进仍在数据同一事务内。
|
|
210
|
+
* lock(E 中-1):写者盖章随成功行保形回写(否则第一窗成功就把章抹了)。
|
|
211
|
+
* ⚠️ resolved_range 的**末端**是「上次成功推进到的时刻」,与删除边界(天对齐下
|
|
212
|
+
* 取 `dayOf(end−1ms)`,见 writeRows)刻意不同:前者是位置、后者是行集覆盖,
|
|
213
|
+
* 混同会让末端出现 −1ms 这种既非水位也非日界的怪值。
|
|
214
|
+
*/
|
|
215
|
+
async function commitWindow(ctx, task, w, kept, expected, watermarkRange, lock) {
|
|
216
|
+
const commit = () => ctx.store.transaction(async (tx) => {
|
|
217
|
+
const written = await writeRows(task, tx, kept, w);
|
|
218
|
+
if (written !== expected) {
|
|
219
|
+
// 红线 2:核对不符即回滚本窗——写一半的状态绝不能连同水位提交。
|
|
220
|
+
throw Object.assign(new Error(`配对核对不符:应写 ${expected} 行,实际写入 ${written} 行`), { code: 'count_mismatch' });
|
|
221
|
+
}
|
|
222
|
+
// null = 本轮尚无连续成功窗(首个窗即失败)→ 不推水位(保持 markPending 的既有
|
|
223
|
+
// 状态),只落数据与清缺口 —— 绝不伪造推进点;下次续跑从头重拉,自愈失败窗。
|
|
224
|
+
if (watermarkRange !== null) {
|
|
225
|
+
await advanceWatermark(tx, task.id, new Date(watermarkRange.end).toISOString(), watermarkRange, new Date(nowOf(ctx)).toISOString(), lock);
|
|
226
|
+
}
|
|
227
|
+
await clearGap(tx, task.id, String(w.start));
|
|
228
|
+
return written;
|
|
229
|
+
});
|
|
230
|
+
return withStoreRetries(ctx, commit);
|
|
231
|
+
}
|
|
232
|
+
/** 假空重试(spec §5.2):空结果重试一次再判空;判空推进 complete(空数据 ≠ 未同步)。 */
|
|
233
|
+
async function fetchWithFakeEmptyRetry(task, ref, ctx) {
|
|
234
|
+
const first = await task.fetch(ref, fetchCtx(ctx));
|
|
235
|
+
if (first.rows.length > 0)
|
|
236
|
+
return first;
|
|
237
|
+
return task.fetch(ref, fetchCtx(ctx));
|
|
238
|
+
}
|
|
239
|
+
function fetchCtx(ctx) {
|
|
240
|
+
// gate/clock 注入(中-1/低-4):任务层远端限频经持久化闸门写冷却、
|
|
241
|
+
// synchronized_at 盖章走注入时钟(与引擎的确定性测试面同源)。
|
|
242
|
+
return { store: ctx.store, credentials: ctx.credentials, gate: ctx.gate, clock: ctx.clock };
|
|
243
|
+
}
|
|
244
|
+
/**
|
|
245
|
+
* 窗口末端上限(G 高-1,任务声明式):`staffAttendance` / `statistic.performance` 的
|
|
246
|
+
* `endTime` 不得晚于今天 00:00(`14501 endTime is invalid`),而「今天 0:00」这条边界
|
|
247
|
+
* 本身不稳定(同一取值时而 200 时而 14501,设计稿 §6.2 实测)→ 统一退到**昨天 00:00**。
|
|
248
|
+
* 返回该任务本轮可拉的末端上限(未声明钳制的任务为 +∞,行为不变)。
|
|
249
|
+
*/
|
|
250
|
+
function windowEndLimit(task, nowMs) {
|
|
251
|
+
return task.windowEndClamp === 'shanghaiYesterday'
|
|
252
|
+
? shanghaiYesterdayStart(nowMs)
|
|
253
|
+
: Number.POSITIVE_INFINITY;
|
|
254
|
+
}
|
|
255
|
+
/**
|
|
256
|
+
* 按上限收边切好的窗:越过上限的窗按上限截断,完全落在上限之后的窗直接丢弃
|
|
257
|
+
*(没有可拉的数据)——宁可少拉一天(次日自动补上),也绝不发一个必被 14501 拒的请求。
|
|
258
|
+
*/
|
|
259
|
+
function clampWindowEnd(windows, limit) {
|
|
260
|
+
if (!Number.isFinite(limit))
|
|
261
|
+
return windows;
|
|
262
|
+
const out = [];
|
|
263
|
+
for (const w of windows) {
|
|
264
|
+
if (w.start >= limit)
|
|
265
|
+
continue;
|
|
266
|
+
out.push(w.end <= limit ? w : { start: w.start, end: limit });
|
|
267
|
+
}
|
|
268
|
+
return out;
|
|
269
|
+
}
|
|
270
|
+
/**
|
|
271
|
+
* 声明了 `windowAlign: 'shanghaiDay'` 的任务的**段式日对齐切片**(修复批 5 #1):
|
|
272
|
+
* 从 `shanghaiDayStart(range.start)`(上海日 0 点)起按 `maxWindowDays` 天的步长推进,
|
|
273
|
+
* 每段 `[d, d + N天)`;**首段钳 start 到 range.start、末段钳 end 到 range.end**(半开
|
|
274
|
+
* `[start, end)` 语义不变,末段 end 恰为 range.end)——除被钳的首尾边界外,每个窗的
|
|
275
|
+
* start/end 都是上海日界。
|
|
276
|
+
*
|
|
277
|
+
* 与 overlapDays 的关系(刻意不设防但写明):对齐只决定窗的**边界**,不改变重叠语义
|
|
278
|
+
* ——overlapDays=0 的报表任务每段互不重叠;若将来有 overlapDays > 0 的任务声明此项,
|
|
279
|
+
* 重叠窗同样落在对齐网格上(`skipContained` 的包含窗判据按真实重叠毫秒数算,不受影响)。
|
|
280
|
+
*
|
|
281
|
+
* 与 `sliceByMaxWindow` 的分界:日对齐只对声明了 `windowAlign` 的任务生效(报表 8 任务);
|
|
282
|
+
* 其余任务(工单 tickets,overlapDays=3、水位 +1ms 任意起点)走原 `sliceByMaxWindow`
|
|
283
|
+
* ——引擎既有任意起点切片结构、3 天回看重叠用例是回归锚。
|
|
284
|
+
*/
|
|
285
|
+
function sliceShanghaiDayAligned(range, maxWindowDays) {
|
|
286
|
+
const stepMs = maxWindowDays * DAY;
|
|
287
|
+
const out = [];
|
|
288
|
+
// 首段网格起点 = 上海日 0 点(可能早于 range.start 最多 <1 天,但下一行的 max
|
|
289
|
+
// 会把窗 start 钳回 range.start —— 产出的首窗 start 恒等于 range.start)。
|
|
290
|
+
let d = shanghaiDayStart(range.start);
|
|
291
|
+
// 防御(理论不可达:调用方保证 start < end 且 range.start ≥ 0):首段起点倒挂时
|
|
292
|
+
// 退回 range.start,绝不产出生成死循环的空步进。
|
|
293
|
+
if (d > range.start)
|
|
294
|
+
d = range.start;
|
|
295
|
+
while (d < range.end) {
|
|
296
|
+
const start = Math.max(d, range.start); // 首段钳到 range.start
|
|
297
|
+
const end = Math.min(d + stepMs, range.end); // 末段钳到 range.end
|
|
298
|
+
out.push({ start, end });
|
|
299
|
+
d += stepMs;
|
|
300
|
+
}
|
|
301
|
+
return out;
|
|
302
|
+
}
|
|
303
|
+
/**
|
|
304
|
+
* 任务感知的切片(runWindowed 与 planRotation 共用,修复批 5 #1 的唯一落点):
|
|
305
|
+
* 声明了 `windowAlign: 'shanghaiDay'` 的任务走段式日对齐;其余走原 `sliceByMaxWindow`。
|
|
306
|
+
* 两处执行路径必须同函数,否则 rotated 与单任务语义分叉。
|
|
307
|
+
* `maxWindowDays` 由调用方在入口校验后传入(两处都有 T2 低-1 的非法值防御)。
|
|
308
|
+
*/
|
|
309
|
+
function sliceForTask(task, range, maxWindowDays) {
|
|
310
|
+
return task.windowAlign === 'shanghaiDay'
|
|
311
|
+
? sliceShanghaiDayAligned(range, maxWindowDays)
|
|
312
|
+
: sliceByMaxWindow(range, maxWindowDays, task.overlapDays ?? 0);
|
|
313
|
+
}
|
|
314
|
+
/**
|
|
315
|
+
* 残段窗判据(修复批 6 #2):日对齐任务切出的窗必须**两端都是上海日界**。
|
|
316
|
+
* 不满足者只可能是被显式区间(--since/--until/--full 的 start=0)首末钳制出的
|
|
317
|
+
* 残段(如 `--since 30d` 的 [今天11:15, 次日0点) 与 [昨天0点, 今天11:15)、
|
|
318
|
+
* `--full` 的 [0, 上海1970-01-01 0点) 16 小时段)。
|
|
319
|
+
*
|
|
320
|
+
* 为什么跳过而不是照拉:这些窗里装的是**半日聚合值**,却会写到 `stat_date =
|
|
321
|
+
* dayOf(start)` 的日行上;同一主键 upsert 会覆盖该日已有的(或将来的)整日聚合行 ——
|
|
322
|
+
* 「每日概览」在该日读到半日值。跳过 = 不 fetch、不写库、不删除、不推水位。
|
|
323
|
+
*
|
|
324
|
+
* 水位语义自检:range = [D0 11:15, D5 11:15],跳过首末残段后成功窗是
|
|
325
|
+
* [D1,D2)…[D4,D5),水位推进到 D5 0:00(末个整日窗 end)——**末残段**(D5 0:00~11:15)
|
|
326
|
+
* 本轮不拉、下轮从 D5 0:00 续跑(日对齐把续跑起点落回日界)拉 [D5, …) 整日:
|
|
327
|
+
* 末残段的整日值**可能**由下轮日对齐 run 补全——前提是该轮的请求区间覆盖 D5 全天
|
|
328
|
+
*(如无 --until 的默认续跑、或 --until 晚于 D6 0 点;`--until D5 11:15` 这类非日界
|
|
329
|
+
* 终点会再次切出残段、继续跳过,不在残段上重复尝试)。**首残段**(D0 11:15~D1 0:00)
|
|
330
|
+
* 不落日行也**不补**——该半日段没有合法落点(写入即半日覆盖整日),属显式 `--since`
|
|
331
|
+
* 区间的边界语义(`windows.skipped` + stderr 进度行如实可见);要覆盖 D0 整日请给
|
|
332
|
+
* 更宽的 `--since`(把起点推到 D0 0 点之前;`resolved_range.start` 的前缀不会自动
|
|
333
|
+
* 前移,回补的唯一途径是显式更宽的区间)。
|
|
334
|
+
* 跳过的窗不算成功窗,`contiguousEnd` 不会越过它
|
|
335
|
+
* (它本来就被排除在循环外,连续成功序列只在整日窗上推进)。
|
|
336
|
+
* 非日对齐任务(工单等任意起点切片)恒返回 false —— 行为不变。
|
|
337
|
+
*/
|
|
338
|
+
function isResidualWindow(task, w) {
|
|
339
|
+
if (task.windowAlign !== 'shanghaiDay')
|
|
340
|
+
return false;
|
|
341
|
+
return w.start !== shanghaiDayStart(w.start) || w.end !== shanghaiDayStart(w.end);
|
|
342
|
+
}
|
|
343
|
+
/** 残段窗的进度行(人读 stderr;skip 窗仍计入 windows.total / windows.skipped)。 */
|
|
344
|
+
const residualNote = (task, w) => `[${task.id}] 残段窗跳过(非日界首/末段):[${new Date(w.start).toISOString()}, ${new Date(w.end).toISOString()})`;
|
|
345
|
+
/**
|
|
346
|
+
* 运行一个同步任务(spec §6.1 的 run(task))。
|
|
347
|
+
* 范围合成:ctx.rangeOverride 优先(--full/--since/--until 的落点)——**显式区间优先于
|
|
348
|
+
* 水位**(B 中-5:--since 是「把这段历史回补一遍」的指令,被水位抬升会让它静默缩成
|
|
349
|
+
* 一窗且输出仍是 complete);无 override 时才走水位续跑 [水位+1ms−overlap, clock 当前],
|
|
350
|
+
* 水位为空从 0 起。已推进过且无新数据 → 空跑 complete。
|
|
351
|
+
* 写者锁(E 中-1):任务开始盖 locked_by/locked_at,结束(含异常)清空。
|
|
352
|
+
*/
|
|
353
|
+
export async function runTask(task, ctx) {
|
|
354
|
+
return task.kind === 'snapshot' ? runSnapshot(task, ctx) : runWindowed(task, ctx);
|
|
355
|
+
}
|
|
356
|
+
/** windowed 任务:读水位 → 切窗 → 逐窗 {gate → fetch → 同事务落库/推水位/清缺口 → 核对}。 */
|
|
357
|
+
async function runWindowed(task, ctx) {
|
|
358
|
+
const { store } = ctx;
|
|
359
|
+
const progress = ctx.onProgress ?? (() => { });
|
|
360
|
+
const gaps = [];
|
|
361
|
+
const now = nowOf(ctx);
|
|
362
|
+
const lock = { by: lockOwner(), at: new Date(now).toISOString() };
|
|
363
|
+
if (task.maxWindowDays === undefined || task.maxWindowDays <= 0) {
|
|
364
|
+
// T2 低-1 防御接线:切分函数对 windowMs ≤ 0 无定义,引擎入口先给出可读错误。
|
|
365
|
+
throw new RangeError(`任务 ${task.id} 的 maxWindowDays 非法:${task.maxWindowDays}(必须为正数)`);
|
|
366
|
+
}
|
|
367
|
+
// —— 范围合成:显式区间优先(B 中-5),否则水位续跑(watermark+1ms − overlapDays 回看)——
|
|
368
|
+
const state = await readWatermark(store, task.id);
|
|
369
|
+
const requested = ctx.rangeOverride ?? { start: 0, end: now };
|
|
370
|
+
const rangeStart = resolveRangeStart(task, ctx, state, requested, now);
|
|
371
|
+
const resolvedRange = { start: rangeStart, end: Math.max(requested.end, rangeStart) };
|
|
372
|
+
if (resolvedRange.end <= resolvedRange.start) {
|
|
373
|
+
return { taskId: task.id, windows: { total: 0, succeeded: 0, failed: 0, skipped: 0 },
|
|
374
|
+
rows: { fetched: 0, written: 0 }, resolvedRange, gaps, status: 'complete' };
|
|
375
|
+
}
|
|
376
|
+
// —— 切窗 + 末端钳制 + 跳过包含窗(T2 裁定接线 / G 高-1 / 修复批 5 #1 日对齐)——
|
|
377
|
+
// 日对齐分支(windowAlign='shanghaiDay')的窗边界全是上海日界:**首窗 start 恒等于
|
|
378
|
+
// rangeStart**(网格起点被 max 钳回,2026-10-17 复审探针 4000 组实测 0 例外)、
|
|
379
|
+
// 末窗 end 恒等于 rangeEnd——对齐只影响**中间**窗的边界。resolved_range.start
|
|
380
|
+
// 仍是水位起点语义(rangeStart),不因对齐回退。
|
|
381
|
+
// 修复批 6 #2:日对齐任务把**残段窗**(非日界首/末段,见 isResidualWindow)排除在
|
|
382
|
+
// 执行集外——runWindowed 与 planRotation 必须同口径(都在切片后过滤)。
|
|
383
|
+
const limit = windowEndLimit(task, now);
|
|
384
|
+
const overlapMs = overlapDaysOf(task);
|
|
385
|
+
// CI 实测修正(2026-10-17):请求区间**整体**早于接口下界(rangeEnd ≤ floor)时,
|
|
386
|
+
// 哨兵起点 0 保持不抬升(floorRangeStart 的既定语义:抬升会把「拿不到的历史」变成
|
|
387
|
+
// 「0 窗 complete」),但**不得按哨兵起点切窗**——[0, now−40d] 会从 1970 按
|
|
388
|
+
// maxWindowDays 切出数百窗(702 窗实测),首窗被任务守卫拒绝、其余各发一次无意义
|
|
389
|
+
// 调用(生产上 opStart 被服务端静默钳到 now−31d,数百窗各拉一遍同一段数据)。
|
|
390
|
+
// 收成单窗直送守卫:显式失败一次、gap 可见,语义仍是「绝不静默成功」。
|
|
391
|
+
const floor = task.windowStartFloorDays !== undefined
|
|
392
|
+
? now - task.windowStartFloorDays * DAY
|
|
393
|
+
: null;
|
|
394
|
+
const wholeRangeBelowFloor = floor !== null && rangeStart <= 0 && resolvedRange.end <= floor;
|
|
395
|
+
const raw = wholeRangeBelowFloor
|
|
396
|
+
? [resolvedRange]
|
|
397
|
+
: sliceForTask(task, resolvedRange, task.maxWindowDays);
|
|
398
|
+
const clamped = clampWindowEnd(raw, limit);
|
|
399
|
+
const skippedResidual = clamped.filter((w) => isResidualWindow(task, w));
|
|
400
|
+
const windows = skipContained(clamped.filter((w) => !isResidualWindow(task, w)), overlapMs);
|
|
401
|
+
// 可拉范围 = 请求范围 ∩ 末端上限:resolved_range 如实回显「这轮实际覆盖到哪」——
|
|
402
|
+
// 回显一个没覆盖到的 end 会让 Agent 以为今天的数据已同步(G 高-1 的静默面)。
|
|
403
|
+
const effectiveRange = { start: rangeStart, end: Math.min(resolvedRange.end, limit) };
|
|
404
|
+
const outRange = { start: rangeStart, end: Math.max(rangeStart, effectiveRange.end) };
|
|
405
|
+
if (windows.length === 0) {
|
|
406
|
+
// 无窗可跑(末端上限之后没有可拉数据,或范围本身为空):**不置 pending**——
|
|
407
|
+
// 状态保持上一轮结论(complete 就是 complete),绝不因为「今天还没有可拉的数据」
|
|
408
|
+
// 把 range_status 打成 pending(verify 会把 pending 判成未完成)。
|
|
409
|
+
// 修复批 6 #2:全部窗都是残段(如区间落在单个非日界日内)时,跳过窗仍如实计数
|
|
410
|
+
//(total = skipped,succeeded/failed 0)——「什么都没做」与「做完了」在输出里可分。
|
|
411
|
+
// 修复批 7 N-1:本路径没有任何成功窗 → resolved_range 与正常路径
|
|
412
|
+
//(`resolvedEndOut ?? outRange.start`)和轮转汇总(`resolvedEnd ?? start`)同口径,
|
|
413
|
+
// 收在起点,**不回显未覆盖的请求区间**——全残段 run 报「覆盖整段」会让 Agent 以为
|
|
414
|
+
// 该段已同步(与 succeeded:0/skipped:total 自相矛盾)。两条子路径:
|
|
415
|
+
// ① 末端上限之后无可拉数据(rangeStart ≥ limit,批 6 之前唯一可达形态)——outRange
|
|
416
|
+
// 此时本就退化为 [start, start](min(end, limit) ≤ limit ≤ start),收口前后等值;
|
|
417
|
+
// ② 全残段跳过(批 6 新增)——outRange 非退化,必须收口到起点。
|
|
418
|
+
for (const w of skippedResidual)
|
|
419
|
+
progress(residualNote(task, w));
|
|
420
|
+
return { taskId: task.id,
|
|
421
|
+
windows: { total: skippedResidual.length, succeeded: 0, failed: 0, skipped: skippedResidual.length },
|
|
422
|
+
rows: { fetched: 0, written: 0 },
|
|
423
|
+
resolvedRange: { start: outRange.start, end: outRange.start }, gaps, status: 'complete' };
|
|
424
|
+
}
|
|
425
|
+
// markPending 只翻状态、保留水位(修复轮中-1):existing 在事务外读出传入;
|
|
426
|
+
// 同时盖写者锁(E 中-1)——与 pending 同事务,避免两次写互相覆盖列。
|
|
427
|
+
await withStoreRetries(ctx, () => store.transaction(async (tx) => {
|
|
428
|
+
await markPending(tx, task.id, state, lock);
|
|
429
|
+
}));
|
|
430
|
+
try {
|
|
431
|
+
let succeeded = 0;
|
|
432
|
+
let failed = 0;
|
|
433
|
+
let fetchedTotal = 0;
|
|
434
|
+
let writtenTotal = 0;
|
|
435
|
+
// —— 水位推进上限(中-2②:失败窗不再被水位永久越过)——
|
|
436
|
+
// contiguousEnd = 最后一个连续成功窗的 end。连续成功序列上每窗成功即推进到本窗
|
|
437
|
+
// 末;一旦出现失败窗(gate 等待超限/fetch/落库),序列断链、水位冻结在断链前
|
|
438
|
+
// 的末端(后续窗仍照常尝试执行、数据仍落库,但水位不越过失败窗)——
|
|
439
|
+
// 增量续跑从冻结处重拉失败窗,gap 可自愈。
|
|
440
|
+
let contiguousEnd = null;
|
|
441
|
+
let contiguousBroken = false;
|
|
442
|
+
// resolved_range 的末端(与水位同源,见循环内注释):逐窗推进;断链时冻结在
|
|
443
|
+
// 连续成功末端(可能为 null = 本轮无任何成功窗 → 回显到连续成功起点为止/未覆盖)。
|
|
444
|
+
let resolvedEndOut = null;
|
|
445
|
+
// 修复批 6 #2:残段窗不进执行循环(上面的过滤已排除),但仍如实计数 ——
|
|
446
|
+
// 裁定:计入 total、标 skipped,每窗一条进度行。「跳过了什么」必须可见,
|
|
447
|
+
// 绝不静默消失;跳过段不算成功窗,contiguousEnd 不受影响(不进循环)。
|
|
448
|
+
const skipped = skippedResidual.length;
|
|
449
|
+
for (const w of skippedResidual)
|
|
450
|
+
progress(residualNote(task, w));
|
|
451
|
+
for (let i = 0; i < windows.length; i++) {
|
|
452
|
+
const w = windows[i];
|
|
453
|
+
const ref = { ...w, index: i, total: windows.length };
|
|
454
|
+
progress(`[${task.id}] 窗口 ${i + 1}/${windows.length}: ${new Date(w.start).toISOString()} ~ ${new Date(w.end).toISOString()}`);
|
|
455
|
+
// —— 限频闸门(持久化冷却;慢不是错:等待后重试本窗,预算耗尽才记 gap)——
|
|
456
|
+
try {
|
|
457
|
+
await acquireGated(ctx, task);
|
|
458
|
+
}
|
|
459
|
+
catch (e) {
|
|
460
|
+
if (e.kind === 'rate_limit_local') {
|
|
461
|
+
failed++;
|
|
462
|
+
contiguousBroken = true;
|
|
463
|
+
await storeGap(ctx, task, w, 'rate_limit_local', e.message, gaps);
|
|
464
|
+
continue;
|
|
465
|
+
}
|
|
466
|
+
throw e;
|
|
467
|
+
}
|
|
468
|
+
// —— fetch(假空重试一次;失败有界重试后记 gap 继续下一窗)——
|
|
469
|
+
let fetchRes;
|
|
470
|
+
try {
|
|
471
|
+
fetchRes = await withRetries(ctx, () => fetchWithFakeEmptyRetry(task, ref, ctx), isTransient);
|
|
472
|
+
}
|
|
473
|
+
catch (e) {
|
|
474
|
+
failed++;
|
|
475
|
+
contiguousBroken = true;
|
|
476
|
+
await storeGap(ctx, task, w, 'fetch_failed', failDetail('取数失败', e), gaps);
|
|
477
|
+
continue;
|
|
478
|
+
}
|
|
479
|
+
// —— enrich 过滤 + 同事务落库(红线 2/3)——
|
|
480
|
+
// 配对核对基准:fetch 声明的 apiCount(brief 钉死用例的权威口径——enrich 淘汰
|
|
481
|
+
// 计入 fetched 但仍参与核对,written < apiCount 即 count_mismatch)。
|
|
482
|
+
// ⚠️ brief 内联裁决(「apiCount 以过滤后应写行数为准」)与 brief 钉死测试矛盾
|
|
483
|
+
//(测试用 enrich 过滤 mock written<apiCount 并断言 mismatch),按派发约束以测试为权威。
|
|
484
|
+
const { kept } = applyEnrich(task, fetchRes.rows);
|
|
485
|
+
fetchedTotal += fetchRes.rows.length;
|
|
486
|
+
const expected = fetchRes.apiCount !== undefined ? fetchRes.apiCount : kept.length;
|
|
487
|
+
// —— 推进值与 resolved_range(中-2② + 修复批 4 #2)——
|
|
488
|
+
// 推进值:未断链 = 本窗 end(连续成功序列上前进);断链后冻结在断链前的连续
|
|
489
|
+
// 成功末端(可能为本轮首窗前 → null → 不推水位)。
|
|
490
|
+
const watermarkMs = contiguousBroken ? contiguousEnd : w.end;
|
|
491
|
+
// resolved_range 的**末端**必须与水位同源:断链时水位停在连续成功末端
|
|
492
|
+
// (早于本窗 end),若 resolved_range 仍报本窗末,`sync status` 会谎报覆盖
|
|
493
|
+
// 到失败窗之后 —— 与水位、与实际删除过的区间全都对不上(修复批 4 #2)。
|
|
494
|
+
const watermarkRange = watermarkMs === null
|
|
495
|
+
? null
|
|
496
|
+
: { start: rangeStart, end: watermarkMs };
|
|
497
|
+
let written = 0;
|
|
498
|
+
try {
|
|
499
|
+
written = await withRetries(ctx, () => commitWindow(ctx, task, w, kept, expected, watermarkRange, lock), isTransient);
|
|
500
|
+
}
|
|
501
|
+
catch (e) {
|
|
502
|
+
failed++;
|
|
503
|
+
contiguousBroken = true;
|
|
504
|
+
const isMismatch = e.code === 'count_mismatch';
|
|
505
|
+
// 行数统计口径(brief 钉死用例):mismatch 时 written 记「应写行数」(kept)——
|
|
506
|
+
// 实际行已随事务回滚,计入的是本窗参与核对的落库意图行数。
|
|
507
|
+
writtenTotal += kept.length;
|
|
508
|
+
await storeGap(ctx, task, w, isMismatch ? 'count_mismatch' : 'write_failed', isMismatch ? e.message : failDetail('落库失败', e), gaps);
|
|
509
|
+
continue;
|
|
510
|
+
}
|
|
511
|
+
resolvedEndOut = watermarkMs;
|
|
512
|
+
if (!contiguousBroken)
|
|
513
|
+
contiguousEnd = w.end;
|
|
514
|
+
// —— gapHint(如 order_performance 的 orders:null 筛选失败,spec §5.2):成功窗也落提示 ——
|
|
515
|
+
// T5 裁定(T7 接线):透传 hint 的原始 reason(filter_matched_no_staff 等),
|
|
516
|
+
// 不压成统一 fetch_failed——缺口原因在 sync_gaps 里必须可分派。
|
|
517
|
+
if (fetchRes.gapHint) {
|
|
518
|
+
await storeGap(ctx, task, w, fetchRes.gapHint.reason, gapHintDetail(fetchRes.gapHint), gaps);
|
|
519
|
+
}
|
|
520
|
+
succeeded++;
|
|
521
|
+
writtenTotal += written;
|
|
522
|
+
progress(`[${task.id}] 窗口 ${i + 1}/${windows.length} 完成:拉取 ${fetchRes.rows.length} 行,写入 ${written} 行`);
|
|
523
|
+
}
|
|
524
|
+
return {
|
|
525
|
+
taskId: task.id,
|
|
526
|
+
windows: { total: windows.length + skipped, succeeded, failed, skipped },
|
|
527
|
+
rows: { fetched: fetchedTotal, written: writtenTotal },
|
|
528
|
+
// 末端与水位同源(见上):断链时 = 连续成功末端,未断链 = 本窗 end。
|
|
529
|
+
// ⚠️ 末端是「上次成功推进到的时刻」,与删除边界(天对齐下取 dayOf(end−1ms),
|
|
530
|
+
// 见 writeRows)刻意不同:前者是位置、后者是行集覆盖,混同会让末端出现
|
|
531
|
+
// −1ms 这种既非水位也非日界的怪值。
|
|
532
|
+
resolvedRange: { start: outRange.start, end: resolvedEndOut ?? outRange.start },
|
|
533
|
+
gaps,
|
|
534
|
+
// brief 权威:有任何窗口失败 → partial(单窗失败也 partial,brief 红线 3 钉死用例);
|
|
535
|
+
// 'failed' 留给连窗口循环都无法进入的异常路径(见 runSnapshot 的 gate 拦截分支)。
|
|
536
|
+
status: failed === 0 ? 'complete' : 'partial',
|
|
537
|
+
};
|
|
538
|
+
}
|
|
539
|
+
finally {
|
|
540
|
+
// 写者锁清空(E 中-1):成功、失败、异常三条路径都要清 —— 锁残留会让 status
|
|
541
|
+
// 永远显示「有人在写」。清章只清锁列,水位/状态按最新值回写。
|
|
542
|
+
await releaseLock(ctx, task);
|
|
543
|
+
}
|
|
544
|
+
}
|
|
545
|
+
/**
|
|
546
|
+
* snapshot 任务的 gap 键:失败路径以 resolvedRange 记 gap,而快照的范围恒为 [0, now]
|
|
547
|
+
* → rangeStart = '0'。成功时清同一把键(C 高-1:此前只记不清,gap 永久留存)。
|
|
548
|
+
*/
|
|
549
|
+
const SNAPSHOT_GAP_KEY = '0';
|
|
550
|
+
/** snapshot 任务:单次全量拉取 → 一个事务内全删重插 + 推水位(spec §5.4)。 */
|
|
551
|
+
async function runSnapshot(task, ctx) {
|
|
552
|
+
const { store } = ctx;
|
|
553
|
+
const progress = ctx.onProgress ?? (() => { });
|
|
554
|
+
const gaps = [];
|
|
555
|
+
const now = nowOf(ctx);
|
|
556
|
+
const resolvedRange = { start: 0, end: now };
|
|
557
|
+
const ref = { start: 0, end: now, index: 0, total: 1 };
|
|
558
|
+
const lock = { by: lockOwner(), at: new Date(now).toISOString() };
|
|
559
|
+
try {
|
|
560
|
+
await acquireGated(ctx, task);
|
|
561
|
+
}
|
|
562
|
+
catch (e) {
|
|
563
|
+
if (e.kind === 'rate_limit_local') {
|
|
564
|
+
// 快照也要留下可查的状态行(E 中-1):门都没进就放弃时,若库里连一行都没有,
|
|
565
|
+
// 建一行 pending(盖章后立即清章)——「这个任务本轮被本地冷却拦下了」必须可查,
|
|
566
|
+
// 而不是 status 里什么都看不到。
|
|
567
|
+
const pre = await readWatermark(store, task.id);
|
|
568
|
+
if (pre.watermark === null && pre.rangeStatus === null && pre.lastSyncedAt === null) {
|
|
569
|
+
await withStoreRetries(ctx, () => store.transaction(async (tx) => {
|
|
570
|
+
await markPending(tx, task.id, pre, lock);
|
|
571
|
+
}));
|
|
572
|
+
await releaseLock(ctx, task);
|
|
573
|
+
}
|
|
574
|
+
await storeGap(ctx, task, resolvedRange, 'rate_limit_local', e.message, gaps);
|
|
575
|
+
return snapshotResult(task, resolvedRange, gaps, 0, 0, 0, 1, 'failed');
|
|
576
|
+
}
|
|
577
|
+
throw e;
|
|
578
|
+
}
|
|
579
|
+
progress(`[${task.id}] 快照全量拉取`);
|
|
580
|
+
// 快照也要进「pending + 盖章」状态:status 的 locked 与 range_status 才有真实语义
|
|
581
|
+
//(此前快照直到成功才写 sync_state,失败时那一行根本不存在)。
|
|
582
|
+
const state = await readWatermark(store, task.id);
|
|
583
|
+
await withStoreRetries(ctx, () => store.transaction(async (tx) => {
|
|
584
|
+
await markPending(tx, task.id, state, lock);
|
|
585
|
+
}));
|
|
586
|
+
try {
|
|
587
|
+
let fetchRes;
|
|
588
|
+
try {
|
|
589
|
+
fetchRes = await withRetries(ctx, () => fetchWithFakeEmptyRetry(task, ref, ctx), isTransient);
|
|
590
|
+
}
|
|
591
|
+
catch (e) {
|
|
592
|
+
await storeGap(ctx, task, resolvedRange, 'fetch_failed', failDetail('取数失败', e), gaps);
|
|
593
|
+
return snapshotResult(task, resolvedRange, gaps, 0, 0, 0, 1, 'failed');
|
|
594
|
+
}
|
|
595
|
+
const { kept } = applyEnrich(task, fetchRes.rows);
|
|
596
|
+
const expected = kept.length;
|
|
597
|
+
let written = 0;
|
|
598
|
+
try {
|
|
599
|
+
written = await withStoreRetries(ctx, () => store.transaction(async (tx) => {
|
|
600
|
+
// 全删重插(spec §5.4:快照幂等来自全量重建);delete '1=1' 是可信代码常量。
|
|
601
|
+
await tx.delete(task.table, '1=1');
|
|
602
|
+
const n = await tx.upsert(task.table, kept);
|
|
603
|
+
if (n !== expected) {
|
|
604
|
+
throw Object.assign(new Error(`配对核对不符:应写 ${expected} 行,实际写入 ${n} 行`), { code: 'count_mismatch' });
|
|
605
|
+
}
|
|
606
|
+
await advanceWatermark(tx, task.id, new Date(now).toISOString(), resolvedRange, new Date(nowOf(ctx)).toISOString(), lock);
|
|
607
|
+
// C 高-1:成功即清本任务快照的 gap(键与失败路径的 rangeStart 同源)。
|
|
608
|
+
await clearGap(tx, task.id, SNAPSHOT_GAP_KEY);
|
|
609
|
+
return n;
|
|
610
|
+
}));
|
|
611
|
+
}
|
|
612
|
+
catch (e) {
|
|
613
|
+
const isMismatch = e.code === 'count_mismatch';
|
|
614
|
+
await storeGap(ctx, task, resolvedRange, isMismatch ? 'count_mismatch' : 'write_failed', isMismatch ? e.message : failDetail('落库失败', e), gaps);
|
|
615
|
+
return snapshotResult(task, resolvedRange, gaps, fetchRes.rows.length, 0, 0, 1, 'failed');
|
|
616
|
+
}
|
|
617
|
+
// gapHint(如 dim.staff_group_members 的 staffIdList 形态漂移):成功快照也落提示。
|
|
618
|
+
// T5 裁定(T7 接线):透传 hint 的原始 reason,不压成统一 fetch_failed ——
|
|
619
|
+
// 缺口原因在 sync_gaps 里必须可分派(与 runWindowed 的 gapHint 路径同口径)。
|
|
620
|
+
if (fetchRes.gapHint) {
|
|
621
|
+
await storeGap(ctx, task, resolvedRange, fetchRes.gapHint.reason, gapHintDetail(fetchRes.gapHint), gaps);
|
|
622
|
+
}
|
|
623
|
+
progress(`[${task.id}] 快照完成:拉取 ${fetchRes.rows.length} 行,写入 ${written} 行`);
|
|
624
|
+
return snapshotResult(task, resolvedRange, gaps, fetchRes.rows.length, written, 1, 0, 'complete');
|
|
625
|
+
}
|
|
626
|
+
finally {
|
|
627
|
+
await releaseLock(ctx, task);
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
/**
|
|
631
|
+
* 清写者锁(E 中-1):只清锁列,水位/状态/同步时间按库中最新值回写。
|
|
632
|
+
* 无行(如空范围空跑、或在 markPending 之前就抛错)时不动库 —— 不为清章凭空建一行。
|
|
633
|
+
*/
|
|
634
|
+
async function releaseLock(ctx, task) {
|
|
635
|
+
const st = await readWatermark(ctx.store, task.id);
|
|
636
|
+
if (st.watermark === null && st.rangeStatus === null && st.lastSyncedAt === null)
|
|
637
|
+
return;
|
|
638
|
+
await withStoreRetries(ctx, () => ctx.store.transaction(async (tx) => {
|
|
639
|
+
await clearLock(tx, task.id, st);
|
|
640
|
+
}));
|
|
641
|
+
}
|
|
642
|
+
/** snapshot 结果封装(windows.total 恒 1:快照是一次全量拉取,成功 1 / 失败 1)。 */
|
|
643
|
+
function snapshotResult(task, resolvedRange, gaps, fetched, written, succeeded, failed, status) {
|
|
644
|
+
return {
|
|
645
|
+
taskId: task.id, windows: { total: 1, succeeded, failed, skipped: 0 },
|
|
646
|
+
rows: { fetched, written }, resolvedRange, gaps, status,
|
|
647
|
+
};
|
|
648
|
+
}
|
|
649
|
+
/**
|
|
650
|
+
* 跳过被前一窗完全覆盖的补窗(T2 裁定:[62,63]⊂[60,63] 场景)。
|
|
651
|
+
* 判据:cur 完全落在 prev 内(cur.end <= prev.end)且起点进入 prev 的重叠端
|
|
652
|
+
*(cur.start >= prev.end − overlapMs)——cur 覆盖的数据 prev 已拉取(重叠保证),
|
|
653
|
+
* 跳过不改变数据、只省一次 API 调用。相邻重叠窗(end 超出 prev.end)不跳过。
|
|
654
|
+
*/
|
|
655
|
+
function skipContained(windows, overlapMs) {
|
|
656
|
+
if (windows.length < 2 || overlapMs <= 0)
|
|
657
|
+
return windows;
|
|
658
|
+
const out = [windows[0]];
|
|
659
|
+
for (let i = 1; i < windows.length; i++) {
|
|
660
|
+
const prev = out[out.length - 1];
|
|
661
|
+
const cur = windows[i];
|
|
662
|
+
if (cur.end <= prev.end && cur.start >= prev.end - overlapMs)
|
|
663
|
+
continue;
|
|
664
|
+
out.push(cur);
|
|
665
|
+
}
|
|
666
|
+
return out;
|
|
667
|
+
}
|
|
668
|
+
async function planRotation(task, ctx, now) {
|
|
669
|
+
if (task.kind !== 'windowed')
|
|
670
|
+
throw new RangeError(`轮转调度只接受 windowed 任务:${task.id}`);
|
|
671
|
+
if (task.maxWindowDays === undefined || task.maxWindowDays <= 0) {
|
|
672
|
+
throw new RangeError(`任务 ${task.id} 的 maxWindowDays 非法:${task.maxWindowDays}(必须为正数)`);
|
|
673
|
+
}
|
|
674
|
+
const state = await readWatermark(ctx.store, task.id);
|
|
675
|
+
const requested = ctx.rangeOverride ?? { start: 0, end: now };
|
|
676
|
+
const rangeStart = resolveRangeStart(task, ctx, state, requested, now);
|
|
677
|
+
const resolvedRange = { start: rangeStart, end: Math.max(requested.end, rangeStart) };
|
|
678
|
+
const overlapMs = overlapDaysOf(task);
|
|
679
|
+
// 末端钳制与 runWindowed 同口径(G 高-1):声明了 windowEndClamp 的任务(考勤/绩效
|
|
680
|
+
// 都是轮转任务)在轮转路径上同样不得发出越界 endTime —— 此前这里少传了一个参数
|
|
681
|
+
// (clampWindowEnd 收 (windows, limit)),等于轮转路径完全没有钳制。
|
|
682
|
+
const limit = windowEndLimit(task, now);
|
|
683
|
+
// 残段窗过滤(修复批 6 #2):与 runWindowed 同口径 —— 切片+钳制后把非日界首/末段
|
|
684
|
+
// 排除在执行集外、收进 skippedResidual(半日聚合不落日行的裁定对轮转路径同样生效)。
|
|
685
|
+
const clamped = resolvedRange.end <= resolvedRange.start
|
|
686
|
+
? []
|
|
687
|
+
: clampWindowEnd(sliceForTask(task, resolvedRange, task.maxWindowDays), limit);
|
|
688
|
+
const skippedResidual = clamped.filter((w) => isResidualWindow(task, w));
|
|
689
|
+
const windows = skipContained(clamped.filter((w) => !isResidualWindow(task, w)), overlapMs);
|
|
690
|
+
const outRange = { start: rangeStart,
|
|
691
|
+
end: Math.max(rangeStart, Math.min(resolvedRange.end, limit)) };
|
|
692
|
+
const lock = { by: lockOwner(), at: new Date(now).toISOString() };
|
|
693
|
+
// 无窗可跑(末端上限之后没有可拉数据 / 范围本身为空):**不置 pending**(与
|
|
694
|
+
// runWindowed 同口径)——状态保持上一轮结论,绝不把 complete 打成 pending
|
|
695
|
+
//(verify 会把 pending 判成未完成)。
|
|
696
|
+
if (windows.length === 0) {
|
|
697
|
+
for (const w of skippedResidual)
|
|
698
|
+
ctx.onProgress?.(residualNote(task, w));
|
|
699
|
+
return { task, state, resolvedRange: outRange, windows, skippedResidual, lock, gaps: [],
|
|
700
|
+
succeeded: 0, failed: 0, fetched: 0, written: 0,
|
|
701
|
+
contiguousEnd: null, contiguousBroken: false, resolvedEnd: null };
|
|
702
|
+
}
|
|
703
|
+
for (const w of skippedResidual)
|
|
704
|
+
ctx.onProgress?.(residualNote(task, w));
|
|
705
|
+
await withStoreRetries(ctx, () => ctx.store.transaction(async (tx) => {
|
|
706
|
+
await markPending(tx, task.id, state, lock);
|
|
707
|
+
}));
|
|
708
|
+
return { task, state, resolvedRange: outRange, windows, skippedResidual, lock, gaps: [],
|
|
709
|
+
succeeded: 0, failed: 0, fetched: 0, written: 0,
|
|
710
|
+
contiguousEnd: null, contiguousBroken: false, resolvedEnd: null };
|
|
711
|
+
}
|
|
712
|
+
/**
|
|
713
|
+
* 报表按天轮转(spec §5.2 实测策略):限频按路径独立 30 秒——同一天(第 i 窗)
|
|
714
|
+
* 内跨任务背靠背调用,不同任务之间不共享冷却;逐任务串行推进则会在同一路径上
|
|
715
|
+
* 连发吃满 31 天 × 30 秒 ≈ 2.6 小时,轮转降到 ≈ 16 分钟。
|
|
716
|
+
*
|
|
717
|
+
* 调度形态:round r = 所有任务的第 r 个窗口背靠背跑完,再进 round r+1。
|
|
718
|
+
* 每任务内部仍复用单任务引擎的全部语义(gate → 假空重试 → 同事务落库/推水位/
|
|
719
|
+
* 清缺口 → 配对核对 → gapHint/gap 记录 + 失败窗有界重试 + 水位冻结),
|
|
720
|
+
* 仅执行顺序按「天分组、组内背靠背」交错。单任务请用 runTask;本函数面向
|
|
721
|
+
* REPORT_TASKS 这类 reportDayRotation 批量入口。
|
|
722
|
+
* 写者锁(E 中-1):每个任务的锁在批次开始时盖上、批次结束(含异常)统一清掉。
|
|
723
|
+
*/
|
|
724
|
+
export async function runRotated(tasks, ctx) {
|
|
725
|
+
const progress = ctx.onProgress ?? (() => { });
|
|
726
|
+
const now = nowOf(ctx);
|
|
727
|
+
const plans = [];
|
|
728
|
+
for (const task of tasks)
|
|
729
|
+
plans.push(await planRotation(task, ctx, now));
|
|
730
|
+
try {
|
|
731
|
+
const maxRounds = plans.reduce((m, p) => Math.max(m, p.windows.length), 0);
|
|
732
|
+
// —— round r:各任务第 r 窗背靠背 ——
|
|
733
|
+
for (let round = 0; round < maxRounds; round++) {
|
|
734
|
+
for (const plan of plans) {
|
|
735
|
+
const { task } = plan;
|
|
736
|
+
const windows = plan.windows;
|
|
737
|
+
if (round >= windows.length)
|
|
738
|
+
continue;
|
|
739
|
+
const w = windows[round];
|
|
740
|
+
const ref = { ...w, index: round, total: windows.length };
|
|
741
|
+
progress(`[${task.id}] 轮转 round ${round + 1}/${maxRounds}: ${new Date(w.start).toISOString()} ~ ${new Date(w.end).toISOString()}`);
|
|
742
|
+
// 限频闸门(同 runWindowed:慢不是错,等待后重试,预算耗尽才记 gap)
|
|
743
|
+
try {
|
|
744
|
+
await acquireGated(ctx, task);
|
|
745
|
+
}
|
|
746
|
+
catch (e) {
|
|
747
|
+
if (e.kind === 'rate_limit_local') {
|
|
748
|
+
plan.failed++;
|
|
749
|
+
plan.contiguousBroken = true;
|
|
750
|
+
await storeGap(ctx, task, w, 'rate_limit_local', e.message, plan.gaps);
|
|
751
|
+
continue;
|
|
752
|
+
}
|
|
753
|
+
throw e;
|
|
754
|
+
}
|
|
755
|
+
// fetch(假空重试一次;失败有界重试后记 gap 继续)
|
|
756
|
+
let fetchRes;
|
|
757
|
+
try {
|
|
758
|
+
fetchRes = await withRetries(ctx, () => fetchWithFakeEmptyRetry(task, ref, ctx), isTransient);
|
|
759
|
+
}
|
|
760
|
+
catch (e) {
|
|
761
|
+
plan.failed++;
|
|
762
|
+
plan.contiguousBroken = true;
|
|
763
|
+
await storeGap(ctx, task, w, 'fetch_failed', failDetail('取数失败', e), plan.gaps);
|
|
764
|
+
continue;
|
|
765
|
+
}
|
|
766
|
+
const { kept } = applyEnrich(task, fetchRes.rows);
|
|
767
|
+
plan.fetched += fetchRes.rows.length;
|
|
768
|
+
const expected = fetchRes.apiCount !== undefined ? fetchRes.apiCount : kept.length;
|
|
769
|
+
// 推进值(中-2②,同 runWindowed):未断链 = 本窗 end;断链后冻结在连续成功末端
|
|
770
|
+
//(可能为 null → 不推水位)。区间起点与 runWindowed 同源(plan.resolvedRange.start)。
|
|
771
|
+
const watermarkMs = plan.contiguousBroken ? plan.contiguousEnd : w.end;
|
|
772
|
+
const watermarkRange = watermarkMs === null
|
|
773
|
+
? null
|
|
774
|
+
: { start: plan.resolvedRange.start, end: watermarkMs };
|
|
775
|
+
try {
|
|
776
|
+
const written = await withRetries(ctx, () => commitWindow(ctx, task, w, kept, expected, watermarkRange, plan.lock), isTransient);
|
|
777
|
+
plan.written += written;
|
|
778
|
+
plan.succeeded++;
|
|
779
|
+
plan.resolvedEnd = watermarkMs;
|
|
780
|
+
if (!plan.contiguousBroken)
|
|
781
|
+
plan.contiguousEnd = w.end;
|
|
782
|
+
}
|
|
783
|
+
catch (e) {
|
|
784
|
+
plan.failed++;
|
|
785
|
+
plan.contiguousBroken = true;
|
|
786
|
+
const isMismatch = e.code === 'count_mismatch';
|
|
787
|
+
plan.written += kept.length;
|
|
788
|
+
await storeGap(ctx, task, w, isMismatch ? 'count_mismatch' : 'write_failed', isMismatch ? e.message : failDetail('落库失败', e), plan.gaps);
|
|
789
|
+
continue;
|
|
790
|
+
}
|
|
791
|
+
if (fetchRes.gapHint) {
|
|
792
|
+
await storeGap(ctx, task, w, fetchRes.gapHint.reason, gapHintDetail(fetchRes.gapHint), plan.gaps);
|
|
793
|
+
}
|
|
794
|
+
progress(`[${task.id}] 轮转 round ${round + 1}/${maxRounds} 完成:拉取 ${fetchRes.rows.length} 行,写入 ${kept.length} 行`);
|
|
795
|
+
}
|
|
796
|
+
}
|
|
797
|
+
}
|
|
798
|
+
finally {
|
|
799
|
+
// 批次结束(含异常)统一清章:每个任务的锁都由本函数盖上,也必须由本函数清掉。
|
|
800
|
+
for (const plan of plans)
|
|
801
|
+
await releaseLock(ctx, plan.task);
|
|
802
|
+
}
|
|
803
|
+
// —— 汇总(空窗任务 total=0、complete,与 runWindowed 空跑语义一致)——
|
|
804
|
+
// T13 接线(T7 报告遗留):resolvedRange 回显各任务的真实合成范围,不再是粗粒度汇总。
|
|
805
|
+
// 末端与水位同源(修复批 4 #2):断链时 = 连续成功末端(resolvedEnd),未断链 = 请求末。
|
|
806
|
+
// ⚠️ 末端是「上次成功推进到的时刻」,与删除边界(天对齐下取 dayOf(end−1ms),
|
|
807
|
+
// 见 writeRows)刻意不同:前者是位置、后者是行集覆盖。
|
|
808
|
+
return plans.map((p) => ({
|
|
809
|
+
taskId: p.task.id,
|
|
810
|
+
// 修复批 6 #2:残段窗计入 total、标 skipped(与 runWindowed 同口径)。
|
|
811
|
+
windows: { total: p.windows.length + p.skippedResidual.length,
|
|
812
|
+
succeeded: p.succeeded, failed: p.failed, skipped: p.skippedResidual.length },
|
|
813
|
+
rows: { fetched: p.fetched, written: p.written },
|
|
814
|
+
resolvedRange: { start: p.resolvedRange.start, end: p.resolvedEnd ?? p.resolvedRange.start },
|
|
815
|
+
gaps: p.gaps,
|
|
816
|
+
status: p.failed === 0 ? 'complete' : 'partial',
|
|
817
|
+
}));
|
|
818
|
+
}
|