qiyu-cli 0.2.1 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +345 -56
- package/dist/cli.js +48 -7
- package/dist/commands/api-notes.js +54 -0
- package/dist/commands/api.js +112 -38
- package/dist/commands/auth.js +150 -28
- package/dist/commands/credentials-source.js +30 -5
- package/dist/commands/db-notes.js +75 -0
- package/dist/commands/db.js +112 -113
- package/dist/commands/query-notes.js +58 -0
- package/dist/commands/query.js +427 -56
- package/dist/commands/skill.js +445 -51
- package/dist/commands/spec-types.js +101 -1
- package/dist/commands/stats.js +139 -30
- package/dist/commands/store-flags.js +18 -0
- package/dist/commands/store-notes.js +38 -0
- package/dist/commands/store-precheck.js +254 -0
- package/dist/commands/store.js +128 -75
- package/dist/commands/sync-notes.js +367 -0
- package/dist/commands/sync-run.js +155 -44
- package/dist/commands/sync.js +181 -120
- package/dist/core/config.js +274 -24
- package/dist/core/credentials.js +38 -4
- package/dist/core/edition.js +1 -1
- package/dist/core/envelope.js +73 -9
- package/dist/core/errors.js +73 -16
- package/dist/core/http.js +151 -52
- package/dist/core/ratelimit.js +5 -1
- package/dist/core/registry/index.js +26 -2
- package/dist/core/registry/params/export.session.check.json +17 -2
- package/dist/core/registry/params/export.session.json +24 -2
- package/dist/core/registry/params/export.session.one.json +10 -2
- package/dist/core/registry/params/export.session.one.message.json +17 -2
- package/dist/core/registry/specs.json +64 -42
- package/dist/core/shell.js +30 -0
- package/dist/core/time.js +1 -1
- package/dist/core/validate.js +21 -2
- package/dist/output.js +204 -28
- package/dist/postinstall.js +14 -5
- package/dist/query/errors.js +145 -45
- package/dist/query/mask.js +1277 -25
- package/dist/query/metric-catalog.js +11 -0
- package/dist/query/native-error.js +189 -0
- package/dist/query/sandbox.js +129 -24
- package/dist/query/table-query.js +36 -5
- package/dist/schedule/install.js +136 -13
- package/dist/skill/check.js +11 -6
- package/dist/skill/detect.js +89 -29
- package/dist/skill/install.js +233 -43
- package/dist/skill/paths.js +54 -0
- package/dist/skill/qiyu/SKILL.md +29 -5
- package/dist/skill/qiyu/errors.md +5 -4
- package/dist/skill/qiyu/protocols.md +57 -5
- package/dist/skill/qiyu/setup.md +6 -2
- package/dist/store/config.js +92 -16
- package/dist/store/duckdb.js +51 -5
- package/dist/store/retry.js +92 -4
- package/dist/store/schema.js +26 -3
- package/dist/sync/engine.js +257 -65
- package/dist/sync/registry.js +4 -0
- package/dist/sync/state.js +20 -0
- package/dist/sync/tasks/dim.js +23 -14
- package/dist/sync/tasks/knowledge.js +50 -8
- package/dist/sync/tasks/pipeline.js +29 -9
- package/dist/sync/tasks/reports.js +176 -49
- package/dist/sync/tasks/sessions.js +88 -21
- package/dist/sync/tasks/tickets.js +59 -8
- package/dist/sync/types.js +60 -1
- package/dist/sync/windowing.js +0 -22
- package/dist/sync/write-contract.js +47 -0
- package/package.json +1 -1
package/dist/sync/engine.js
CHANGED
|
@@ -6,12 +6,27 @@ import { shanghaiDayIso, shanghaiDayStart, shanghaiYesterdayStart } from './day.
|
|
|
6
6
|
const DAY = 86_400_000;
|
|
7
7
|
/** 回看毫秒数:水位续跑的起点回退量(overlapDays 天;0 = 不回看)。 */
|
|
8
8
|
const overlapDaysOf = (task) => (task.overlapDays ?? 0) * DAY;
|
|
9
|
+
/**
|
|
10
|
+
* 窗口级 gap 的 reason —— **完整值域 = 引擎侧失败原因 ∪ 任务层业务提示**
|
|
11
|
+
* (`sync_gaps.reason` / `gaps[].reason` 的取值都来自这两组)。
|
|
12
|
+
*
|
|
13
|
+
* ⚠️ 2026-10-27 修:此前 `WindowGapReason` 只声明引擎侧四个,任务 `gapHint.reason`
|
|
14
|
+
* 是 `string` 且落库处用 `as WindowGapReason` 强转 —— 类型对运行时谎报(业务 reason
|
|
15
|
+
* 真实存在于库里),且这些值全仓无图例。现在两组都是 `as const` 数组(可导出、
|
|
16
|
+
* 可被测试逐项核对),README 图例由 `tests/sync-enums.test.ts` 双向钉住。
|
|
17
|
+
*/
|
|
18
|
+
export const ENGINE_GAP_REASONS = [
|
|
19
|
+
'count_mismatch', // 写入意图行数 ≠ API 声明行数(事务已回滚)
|
|
20
|
+
'fetch_failed', // 取数失败(有界重试后仍失败,本窗未写入)
|
|
21
|
+
'write_failed', // 落库失败(事务已回滚,本窗未写入)
|
|
22
|
+
'rate_limit_local', // 本地限频冷却未到点(持久化冷却)
|
|
23
|
+
];
|
|
9
24
|
/** 盖在 sync_state.locked_by 上的进程标识(E 中-1:status 的 locked 字段据此为真)。 */
|
|
10
25
|
const lockOwner = () => `pid ${process.pid}`;
|
|
11
26
|
/** 引擎自己的「现在」:ctx.now 优先(命令层给任务层固定 clock 时引擎仍可用真实时刻)。 */
|
|
12
27
|
const nowOf = (ctx) => (ctx.now ?? ctx.clock)();
|
|
13
|
-
/** 限频闸门的键(B 中-6
|
|
14
|
-
const
|
|
28
|
+
/** 限频闸门的键(B 中-6):任务声明的全部 endpoint,缺省回退 `endpoint ?? id`。 */
|
|
29
|
+
const gateKeysOf = (task) => task.gateEndpoints !== undefined ? [...task.gateEndpoints] : [task.endpoint ?? task.id];
|
|
15
30
|
/**
|
|
16
31
|
* 首跑起点的合理下界(C 中-2):起点是 0(或负数)时它是「从头开始」的**哨兵**,
|
|
17
32
|
* 不是真实日期 —— 1970 年没有任何数据,而 `ticket.search` 的 `opStart` 还会被服务端
|
|
@@ -67,25 +82,40 @@ function failDetail(label, e) {
|
|
|
67
82
|
return `${label}(底层:${raw})`;
|
|
68
83
|
}
|
|
69
84
|
/**
|
|
70
|
-
* 写策略落地(事务内调用):deleteRangeInsert
|
|
71
|
-
* ⚠️ 窗口是**半开** [start, end),而删除条件是闭区间 ——
|
|
72
|
-
* `dayIso(end − 1ms)`(窗口内最后一个瞬时所在的自然日),否则 end
|
|
73
|
-
|
|
74
|
-
*
|
|
75
|
-
*
|
|
76
|
-
*
|
|
77
|
-
*
|
|
85
|
+
* 写策略落地(事务内调用):deleteRangeInsert 按声明基准删旧 + upsert;upsertRows 直接 upsert。
|
|
86
|
+
* ⚠️ 窗口是**半开** [start, end),而删除条件是闭区间 —— 上界一律取「最后 1ms」:
|
|
87
|
+
* · 天对齐分支取 `dayIso(end − 1ms)`(窗口内最后一个瞬时所在的自然日),否则 end 恰落在
|
|
88
|
+
* 日界时(`--until` 指定日期 / windowEndClamp 钳到昨天 0:00 / 日对齐水位都会产生)
|
|
89
|
+
* `dayIso(end)` = 下一天零点,删除会多删**下一天**的行:`--since D0 --until D2` 的第二窗
|
|
90
|
+
* end=D2 零点 → 删除范围含 D2 → D2 日已同步的行被删而本轮不重拉(D2 在请求区间外,
|
|
91
|
+
* 永久损失)。
|
|
92
|
+
* · `instantColumn` 分支取 `toISOString(end − 1ms)`,与行内值同为毫秒精度 ISO 串,
|
|
93
|
+
* 等价于 `< end`。
|
|
78
94
|
*/
|
|
79
95
|
async function writeRows(task, ops, rows, range) {
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
96
|
+
const write = task.write;
|
|
97
|
+
if (write.kind === 'deleteRangeInsert') {
|
|
98
|
+
if ('instantColumn' in write) {
|
|
99
|
+
// ⚠️ 行内自带**逐行瞬时时间**的表按 `[start, end)` 精确删(2026-11-03 修,CodeRabbit #59):
|
|
100
|
+
// 按自然日删在「窗口起止不在日界」时无解 —— 删整日会把相邻窗写好的边界日整片抹掉
|
|
101
|
+
// (实测 staff_attendance 边界日 16/24 小时);改成「只删完整日内」又会把半个边界日里
|
|
102
|
+
// 被上游撤掉的行永久留下(陈旧行)。逐行时间列能同时避免两头:窗内精确覆盖、窗外不碰。
|
|
103
|
+
// 列名来自任务声明(代码级),值参数化绑定;`write-contract.ts` 保证它是表的
|
|
104
|
+
// timeColumn(kind='iso-instant'),与 `toISOString()` 的比较口径一致。
|
|
105
|
+
await ops.delete(task.table, `"${write.instantColumn}" >= ? AND "${write.instantColumn}" <= ?`, [new Date(range.start).toISOString(), new Date(range.end - 1).toISOString()]);
|
|
106
|
+
}
|
|
107
|
+
else {
|
|
108
|
+
const dayAligned = write.rangeAlign === 'shanghaiDay';
|
|
109
|
+
const from = dayAligned ? shanghaiDayIso(range.start) : new Date(range.start).toISOString();
|
|
110
|
+
const to = dayAligned ? shanghaiDayIso(range.end - 1) : new Date(range.end).toISOString();
|
|
111
|
+
// where 是可信代码常量(同 query 红线):列名来自任务声明(代码级),值参数化绑定。
|
|
112
|
+
// 范围边界与行内日期列同口径(rangeAlign):行写「上海天对齐的日期列」时,删除范围
|
|
113
|
+
// 也必须是天对齐边界——否则 --full 与增量对同一业务日的删除范围互不覆盖(A 中-2)。
|
|
114
|
+
// ⚠️ 按自然日删**只在窗起止都落在日界时精确**:注册表加载期由 `write-contract.ts`
|
|
115
|
+
// 校验「rangeAlign='shanghaiDay' ⇒ windowAlign='shanghaiDay'」,不让边界日被邻窗抹掉。
|
|
116
|
+
for (const col of write.rangeColumns) {
|
|
117
|
+
await ops.delete(task.table, `"${col}" >= ? AND "${col}" <= ?`, [from, to]);
|
|
118
|
+
}
|
|
89
119
|
}
|
|
90
120
|
}
|
|
91
121
|
if (rows.length === 0)
|
|
@@ -140,6 +170,11 @@ async function withRetries(ctx, op, retryable = () => true) {
|
|
|
140
170
|
* · 网络失败(kind=network)**不在此列** —— core `http.call` 已自带 2 次退避重试
|
|
141
171
|
* (实测 300ms/600ms),引擎再套一层等于 3 倍往返与 3 倍等待,把 core 的预算语义
|
|
142
172
|
* 变成噪声(实测形态:一次「fetch failed」会从 3 次请求变成 9 次);
|
|
173
|
+
* ⚠️ 这条前提在 2026-11-02 之前**是假的**:当时只有 fetch 抛错与 body 读取失败
|
|
174
|
+
* 两支会重试,5xx(网关 HTML / Spring JSON 两种载体)**只发 1 个请求就抛出** ——
|
|
175
|
+
* 于是一次网关 502 就把整个窗口打成 `fetch_failed` gap,而 gap 的 detail 还写着
|
|
176
|
+
* 「可退避重试」。现在 core 的四支网络载体共用同一预算(实测每支 3 次请求),
|
|
177
|
+
* 前提才成立;改 core 的重试覆盖面时必须一并回来核对这条注释。
|
|
143
178
|
* · 限频(kind=rate_limit)不在此列 —— 任务层 callPaced 已有自己的等待预算,
|
|
144
179
|
* 引擎层再等一轮只会把冷却乘起来(限频靠冷却解决,不靠重试);
|
|
145
180
|
* · 本地参数错误(usage)、配对核对结论(count_mismatch)、任务层自抛的形态漂移
|
|
@@ -185,18 +220,23 @@ function gapHintDetail(hint) {
|
|
|
185
220
|
const GATE_WAIT_BUDGET = 3;
|
|
186
221
|
const GATE_WAIT_FALLBACK_MS = 1_000;
|
|
187
222
|
async function acquireGated(ctx, task) {
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
223
|
+
// ⚠️ 每个声明的 endpoint 都要 acquire(2026-11-02 实测修):`callPaced` 命中远端限频
|
|
224
|
+
// 时 penalize 写的是**那个** endpoint 的冷却行,这里只查 `task.endpoint` 会漏掉
|
|
225
|
+
// 任务实际调用、却被限流的其它端点(ticket.logs 的 ticket.log、dim 的 template.list)。
|
|
226
|
+
// 漏掉的冷却读不到 → 下一窗照旧超发。逐个等,都到点才放行(慢不是错)。
|
|
227
|
+
for (const key of gateKeysOf(task)) {
|
|
228
|
+
for (let waits = 0;; waits++) {
|
|
229
|
+
try {
|
|
230
|
+
// 冷却是持久化的:读库也在 store 上,撞锁走同一有界重试(B 中-4)。
|
|
231
|
+
await withStoreRetries(ctx, () => ctx.gate.acquire(key));
|
|
232
|
+
break;
|
|
233
|
+
}
|
|
234
|
+
catch (e) {
|
|
235
|
+
if (e.kind !== 'rate_limit_local' || waits >= GATE_WAIT_BUDGET)
|
|
236
|
+
throw e;
|
|
237
|
+
const wait = clampGateWait(e.retryAfter ?? 0) || GATE_WAIT_FALLBACK_MS;
|
|
238
|
+
await engineSleep(ctx, wait);
|
|
239
|
+
}
|
|
200
240
|
}
|
|
201
241
|
}
|
|
202
242
|
}
|
|
@@ -243,7 +283,7 @@ function fetchCtx(ctx) {
|
|
|
243
283
|
}
|
|
244
284
|
/**
|
|
245
285
|
* 窗口末端上限(G 高-1,任务声明式):`staffAttendance` / `statistic.performance` 的
|
|
246
|
-
* `endTime`
|
|
286
|
+
* `endTime` 必须早于今天 00:00(`14501 endTime is invalid`),而「今天 0:00」这条边界
|
|
247
287
|
* 本身不稳定(同一取值时而 200 时而 14501,设计稿 §6.2 实测)→ 统一退到**昨天 00:00**。
|
|
248
288
|
* 返回该任务本轮可拉的末端上限(未声明钳制的任务为 +∞,行为不变)。
|
|
249
289
|
*/
|
|
@@ -267,6 +307,24 @@ function clampWindowEnd(windows, limit) {
|
|
|
267
307
|
}
|
|
268
308
|
return out;
|
|
269
309
|
}
|
|
310
|
+
/** 被 clampWindowEnd **整窗丢弃**的窗数(丢弃不等于跳过:必须计数可见,2026-11-03 修)。 */
|
|
311
|
+
const clampedAwayCount = (windows, limit) => Number.isFinite(limit) ? windows.filter((w) => w.start >= limit).length : 0;
|
|
312
|
+
/**
|
|
313
|
+
* 被 clampWindowEnd **截短**(不是整窗丢弃)的窗(2026-11-09 审计)。
|
|
314
|
+
*
|
|
315
|
+
* 它们照常执行,但实际拉取的时间片比切窗时短(如 31 天窗只拉到 `limit`)。
|
|
316
|
+
* 计数上它是「一个成功窗」,`resolved_range.end` 也已如实收到 `limit` —— 唯独
|
|
317
|
+
* 「这个窗被截短了」这一步此前没有任何输出。stderr 进度行补上它(数据没少:
|
|
318
|
+
* 截掉的那段本来就晚于上限、拉不到)。
|
|
319
|
+
*/
|
|
320
|
+
const truncatedByClamp = (windows, limit) => Number.isFinite(limit) ? windows.filter((w) => w.start < limit && w.end > limit) : [];
|
|
321
|
+
/** 合并窗的覆盖窗失败时的进度行:那个窗被合并进了一个没成功的窗 —— 该段没拉到。 */
|
|
322
|
+
const containedUnreachedNote = (task, w) => `[${task.id}] 窗未执行(原本落在前一窗的回看重叠区、被合并,但覆盖它的那一窗失败):`
|
|
323
|
+
+ `[${new Date(w.start).toISOString()}, ${new Date(w.end).toISOString()}) —— 已按 skipped 计入,须重跑补齐`;
|
|
324
|
+
/** 截短窗的进度行(人读 stderr;与整窗丢弃的 clampedAwayNote 分开说成因)。 */
|
|
325
|
+
const clampedShortNote = (task, w, limit) => `[${task.id}] 窗末端被上限截短(只拉到 ${new Date(limit).toISOString()}):`
|
|
326
|
+
+ `[${new Date(w.start).toISOString()}, ${new Date(w.end).toISOString()}) —— `
|
|
327
|
+
+ '截掉的部分晚于末端上限、接口给不出数据,不是丢失已有数据';
|
|
270
328
|
/**
|
|
271
329
|
* 声明了 `windowAlign: 'shanghaiDay'` 的任务的**段式日对齐切片**(修复批 5 #1):
|
|
272
330
|
* 从 `shanghaiDayStart(range.start)`(上海日 0 点)起按 `maxWindowDays` 天的步长推进,
|
|
@@ -341,6 +399,9 @@ function isResidualWindow(task, w) {
|
|
|
341
399
|
return w.start !== shanghaiDayStart(w.start) || w.end !== shanghaiDayStart(w.end);
|
|
342
400
|
}
|
|
343
401
|
/** 残段窗的进度行(人读 stderr;skip 窗仍计入 windows.total / windows.skipped)。 */
|
|
402
|
+
/** 整窗晚于末端上限时的进度行(与残段窗同为「有意不 fetch」,但成因不同,分开说)。 */
|
|
403
|
+
const clampedAwayNote = (task, w, limit) => `[${task.id}] 窗整窗跳过(晚于末端上限 ${new Date(limit).toISOString()}):`
|
|
404
|
+
+ `[${new Date(w.start).toISOString()}, ${new Date(w.end).toISOString()})`;
|
|
344
405
|
const residualNote = (task, w) => `[${task.id}] 残段窗跳过(非日界首/末段):[${new Date(w.start).toISOString()}, ${new Date(w.end).toISOString()})`;
|
|
345
406
|
/**
|
|
346
407
|
* 运行一个同步任务(spec §6.1 的 run(task))。
|
|
@@ -395,9 +456,22 @@ async function runWindowed(task, ctx) {
|
|
|
395
456
|
const raw = wholeRangeBelowFloor
|
|
396
457
|
? [resolvedRange]
|
|
397
458
|
: sliceForTask(task, resolvedRange, task.maxWindowDays);
|
|
459
|
+
// ⚠️ 末端上限之外的窗是**整窗丢弃**(2026-11-03 实测修):此前它既不在 total 也不在
|
|
460
|
+
// skipped、无任何进度行 —— 请求「今天」的考勤/绩效会得到 `total:0, complete, gaps:[]`,
|
|
461
|
+
// Agent 无法区分「今天被上限挡掉了」与「已同步完成」,而本仓自定的残段窗原则是
|
|
462
|
+
// 「绝不静默消失」。现在与残段窗一样计数进 skipped 并逐窗出进度行。
|
|
463
|
+
const droppedByClamp = raw.filter((w) => w.start >= limit);
|
|
398
464
|
const clamped = clampWindowEnd(raw, limit);
|
|
399
465
|
const skippedResidual = clamped.filter((w) => isResidualWindow(task, w));
|
|
400
|
-
const
|
|
466
|
+
const notResidual = clamped.filter((w) => !isResidualWindow(task, w));
|
|
467
|
+
const links = containedLinks(notResidual, overlapMs);
|
|
468
|
+
const windows = links.kept;
|
|
469
|
+
// 重叠压缩掉的窗(2026-11-09 审计):overlapDays > 0 的任务里,完全落进前一窗重叠区的
|
|
470
|
+
// 窗被合并——它既不执行也不该算「跳过」(按切窗形态,数据已被前一窗覆盖),但此前它从
|
|
471
|
+
// 所有计数里消失。⚠️ 是否**真的**覆盖要看覆盖窗的结果(Codex 2026-11-10 第三轮 P2:
|
|
472
|
+
// 覆盖窗自己失败时 `failed:1, contained:1` 而表是空的)—— 所以 `containedCount` 等到
|
|
473
|
+
// 循环结束、按「覆盖窗成功」结算(见下面循环后的结算段),失败覆盖窗名下的合并窗进 skipped。
|
|
474
|
+
// ⚠️ 只能声明在成功返回的那条路径作用域里(空窗早返回不涉及任何反馈射)。
|
|
401
475
|
// 可拉范围 = 请求范围 ∩ 末端上限:resolved_range 如实回显「这轮实际覆盖到哪」——
|
|
402
476
|
// 回显一个没覆盖到的 end 会让 Agent 以为今天的数据已同步(G 高-1 的静默面)。
|
|
403
477
|
const effectiveRange = { start: rangeStart, end: Math.min(resolvedRange.end, limit) };
|
|
@@ -417,8 +491,13 @@ async function runWindowed(task, ctx) {
|
|
|
417
491
|
// ② 全残段跳过(批 6 新增)——outRange 非退化,必须收口到起点。
|
|
418
492
|
for (const w of skippedResidual)
|
|
419
493
|
progress(residualNote(task, w));
|
|
494
|
+
for (const w of droppedByClamp)
|
|
495
|
+
progress(clampedAwayNote(task, w, limit));
|
|
496
|
+
for (const w of truncatedByClamp(raw, limit))
|
|
497
|
+
progress(clampedShortNote(task, w, limit));
|
|
498
|
+
const skippedAll = skippedResidual.length + droppedByClamp.length;
|
|
420
499
|
return { taskId: task.id,
|
|
421
|
-
windows: { total:
|
|
500
|
+
windows: { total: skippedAll, succeeded: 0, failed: 0, skipped: skippedAll },
|
|
422
501
|
rows: { fetched: 0, written: 0 },
|
|
423
502
|
resolvedRange: { start: outRange.start, end: outRange.start }, gaps, status: 'complete' };
|
|
424
503
|
}
|
|
@@ -445,12 +524,21 @@ async function runWindowed(task, ctx) {
|
|
|
445
524
|
// 修复批 6 #2:残段窗不进执行循环(上面的过滤已排除),但仍如实计数 ——
|
|
446
525
|
// 裁定:计入 total、标 skipped,每窗一条进度行。「跳过了什么」必须可见,
|
|
447
526
|
// 绝不静默消失;跳过段不算成功窗,contiguousEnd 不受影响(不进循环)。
|
|
448
|
-
const skipped = skippedResidual.length;
|
|
527
|
+
const skipped = skippedResidual.length + droppedByClamp.length;
|
|
449
528
|
for (const w of skippedResidual)
|
|
450
529
|
progress(residualNote(task, w));
|
|
530
|
+
for (const w of droppedByClamp)
|
|
531
|
+
progress(clampedAwayNote(task, w, limit));
|
|
532
|
+
for (const w of truncatedByClamp(raw, limit))
|
|
533
|
+
progress(clampedShortNote(task, w, limit));
|
|
534
|
+
// 每个执行窗成功与否,结束时结算它名下的合并窗(见上的注释)
|
|
535
|
+
const windowOk = [];
|
|
536
|
+
let containedCount = 0;
|
|
537
|
+
let containedUnreached = 0;
|
|
451
538
|
for (let i = 0; i < windows.length; i++) {
|
|
452
539
|
const w = windows[i];
|
|
453
540
|
const ref = { ...w, index: i, total: windows.length };
|
|
541
|
+
windowOk.push(false);
|
|
454
542
|
progress(`[${task.id}] 窗口 ${i + 1}/${windows.length}: ${new Date(w.start).toISOString()} ~ ${new Date(w.end).toISOString()}`);
|
|
455
543
|
// —— 限频闸门(持久化冷却;慢不是错:等待后重试本窗,预算耗尽才记 gap)——
|
|
456
544
|
try {
|
|
@@ -514,16 +602,38 @@ async function runWindowed(task, ctx) {
|
|
|
514
602
|
// —— gapHint(如 order_performance 的 orders:null 筛选失败,spec §5.2):成功窗也落提示 ——
|
|
515
603
|
// T5 裁定(T7 接线):透传 hint 的原始 reason(filter_matched_no_staff 等),
|
|
516
604
|
// 不压成统一 fetch_failed——缺口原因在 sync_gaps 里必须可分派。
|
|
517
|
-
|
|
518
|
-
|
|
605
|
+
// ⚠️ 单条或**多条**(CodeRabbit PR #67):同一窗可能既缺行(row_missing_key)又缺
|
|
606
|
+
// fixture(export_fixture_unreported),压成一条会让消费方永远漏掉另一类信号。
|
|
607
|
+
const hints = Array.isArray(fetchRes.gapHint) ? fetchRes.gapHint : [fetchRes.gapHint];
|
|
608
|
+
for (const h of hints) {
|
|
609
|
+
if (h === undefined)
|
|
610
|
+
continue;
|
|
611
|
+
await storeGap(ctx, task, w, h.reason, gapHintDetail(h), gaps);
|
|
519
612
|
}
|
|
520
613
|
succeeded++;
|
|
614
|
+
windowOk[i] = true;
|
|
521
615
|
writtenTotal += written;
|
|
522
616
|
progress(`[${task.id}] 窗口 ${i + 1}/${windows.length} 完成:拉取 ${fetchRes.rows.length} 行,写入 ${written} 行`);
|
|
523
617
|
}
|
|
618
|
+
// 合并窗结算:覆盖窗成功 → contained;覆盖窗失败 → 这些窗该拉没拉 → skipped
|
|
619
|
+
//(并逐窗出进度行点名「未覆盖」,绝不静默)。
|
|
620
|
+
for (let i = 0; i < windows.length; i++) {
|
|
621
|
+
const merged = links.containedBy[i] ?? [];
|
|
622
|
+
if (merged.length === 0)
|
|
623
|
+
continue;
|
|
624
|
+
if (windowOk[i])
|
|
625
|
+
containedCount += merged.length;
|
|
626
|
+
else {
|
|
627
|
+
containedUnreached += merged.length;
|
|
628
|
+
for (const w of merged)
|
|
629
|
+
progress(containedUnreachedNote(task, w));
|
|
630
|
+
}
|
|
631
|
+
}
|
|
632
|
+
const skippedTotal = skipped + containedUnreached;
|
|
524
633
|
return {
|
|
525
634
|
taskId: task.id,
|
|
526
|
-
windows: { total: windows.length +
|
|
635
|
+
windows: { total: windows.length + skippedTotal, succeeded, failed, skipped: skippedTotal,
|
|
636
|
+
...(containedCount > 0 ? { contained: containedCount } : {}) },
|
|
527
637
|
rows: { fetched: fetchedTotal, written: writtenTotal },
|
|
528
638
|
// 末端与水位同源(见上):断链时 = 连续成功末端,未断链 = 本窗 end。
|
|
529
639
|
// ⚠️ 末端是「上次成功推进到的时刻」,与删除边界(天对齐下取 dayOf(end−1ms),
|
|
@@ -593,7 +703,12 @@ async function runSnapshot(task, ctx) {
|
|
|
593
703
|
return snapshotResult(task, resolvedRange, gaps, 0, 0, 0, 1, 'failed');
|
|
594
704
|
}
|
|
595
705
|
const { kept } = applyEnrich(task, fetchRes.rows);
|
|
596
|
-
|
|
706
|
+
// ⚠️ 与 windowed(`engine.ts` 的 `expected = apiCount ?? kept.length`)和 `sync verify`
|
|
707
|
+
// (`apiCount ?? rows.length`)**同一口径**:`apiCount` 是 fetch 声明的配对核对基准,
|
|
708
|
+
// 快照此前只用 `kept.length` —— 三条路径对同一字段三种读法,上游声明的行数与实际
|
|
709
|
+
// 解析出的行数不等时,只有快照会把它静默吞掉(`types.ts` 的字段注释写着「配对核对
|
|
710
|
+
// 基准」)。当前所有快照任务都把 apiCount 取成过滤后的行数,故行为零漂移。
|
|
711
|
+
const expected = fetchRes.apiCount !== undefined ? fetchRes.apiCount : kept.length;
|
|
597
712
|
let written = 0;
|
|
598
713
|
try {
|
|
599
714
|
written = await withStoreRetries(ctx, () => store.transaction(async (tx) => {
|
|
@@ -617,8 +732,10 @@ async function runSnapshot(task, ctx) {
|
|
|
617
732
|
// gapHint(如 dim.staff_group_members 的 staffIdList 形态漂移):成功快照也落提示。
|
|
618
733
|
// T5 裁定(T7 接线):透传 hint 的原始 reason,不压成统一 fetch_failed ——
|
|
619
734
|
// 缺口原因在 sync_gaps 里必须可分派(与 runWindowed 的 gapHint 路径同口径)。
|
|
620
|
-
|
|
621
|
-
|
|
735
|
+
for (const h of (Array.isArray(fetchRes.gapHint) ? fetchRes.gapHint : [fetchRes.gapHint])) {
|
|
736
|
+
if (h === undefined)
|
|
737
|
+
continue;
|
|
738
|
+
await storeGap(ctx, task, resolvedRange, h.reason, gapHintDetail(h), gaps);
|
|
622
739
|
}
|
|
623
740
|
progress(`[${task.id}] 快照完成:拉取 ${fetchRes.rows.length} 行,写入 ${written} 行`);
|
|
624
741
|
return snapshotResult(task, resolvedRange, gaps, fetchRes.rows.length, written, 1, 0, 'complete');
|
|
@@ -653,17 +770,47 @@ function snapshotResult(task, resolvedRange, gaps, fetched, written, succeeded,
|
|
|
653
770
|
* 跳过不改变数据、只省一次 API 调用。相邻重叠窗(end 超出 prev.end)不跳过。
|
|
654
771
|
*/
|
|
655
772
|
function skipContained(windows, overlapMs) {
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
773
|
+
return containedLinks(windows, overlapMs).kept;
|
|
774
|
+
}
|
|
775
|
+
/**
|
|
776
|
+
* 与 `skipContained` 同一次遍历,额外记下**每个被合并窗落在哪个执行窗里**。
|
|
777
|
+
*
|
|
778
|
+
* 被合并 ≠ 已覆盖(Codex 2026-11-10 第三轮 P2):覆盖它的那一窗自己也可能失败
|
|
779
|
+
* (`sessions.export` 的 1 天窗失败 → `failed:1, contained:1` 而表是空的)。所以
|
|
780
|
+
* `windows.contained` 只统计「**覆盖窗已经成功**」的那些合并窗,未覆盖的归入 `unreached`,
|
|
781
|
+
* 由调用方按 skipped 计 —— 字段的承诺是「数据已由前一窗覆盖」,不能只看切窗形态。
|
|
782
|
+
*/
|
|
783
|
+
function containedLinks(windows, overlapMs) {
|
|
784
|
+
const kept = [];
|
|
785
|
+
const containedBy = [];
|
|
786
|
+
if (windows.length === 0)
|
|
787
|
+
return { kept, containedBy, unreached: [] };
|
|
788
|
+
if (windows.length < 2 || overlapMs <= 0) {
|
|
789
|
+
kept.push(...windows);
|
|
790
|
+
for (let i = 0; i < windows.length; i++)
|
|
791
|
+
containedBy.push([]);
|
|
792
|
+
return { kept, containedBy, unreached: [] };
|
|
793
|
+
}
|
|
794
|
+
let dropped = [];
|
|
795
|
+
kept.push(windows[0]);
|
|
796
|
+
containedBy.push([]);
|
|
659
797
|
for (let i = 1; i < windows.length; i++) {
|
|
660
|
-
const prev =
|
|
798
|
+
const prev = kept[kept.length - 1];
|
|
661
799
|
const cur = windows[i];
|
|
662
|
-
if (cur.end <= prev.end && cur.start >= prev.end - overlapMs)
|
|
800
|
+
if (cur.end <= prev.end && cur.start >= prev.end - overlapMs) {
|
|
801
|
+
dropped.push(cur);
|
|
663
802
|
continue;
|
|
664
|
-
|
|
803
|
+
}
|
|
804
|
+
containedBy[containedBy.length - 1] = dropped;
|
|
805
|
+
dropped = [];
|
|
806
|
+
kept.push(cur);
|
|
807
|
+
containedBy.push([]);
|
|
665
808
|
}
|
|
666
|
-
|
|
809
|
+
// 最后一个执行窗之后若还有被合并窗(cur 落在 prev 重叠区但 prev 已是末窗)——
|
|
810
|
+
// 上面的循环只在遇到下一个执行窗时才结算 dropped,末尾这次要补上
|
|
811
|
+
if (dropped.length > 0)
|
|
812
|
+
containedBy[containedBy.length - 1] = dropped;
|
|
813
|
+
return { kept, containedBy, unreached: [] };
|
|
667
814
|
}
|
|
668
815
|
async function planRotation(task, ctx, now) {
|
|
669
816
|
if (task.kind !== 'windowed')
|
|
@@ -682,11 +829,20 @@ async function planRotation(task, ctx, now) {
|
|
|
682
829
|
const limit = windowEndLimit(task, now);
|
|
683
830
|
// 残段窗过滤(修复批 6 #2):与 runWindowed 同口径 —— 切片+钳制后把非日界首/末段
|
|
684
831
|
// 排除在执行集外、收进 skippedResidual(半日聚合不落日行的裁定对轮转路径同样生效)。
|
|
685
|
-
const
|
|
832
|
+
const rawWindows = resolvedRange.end <= resolvedRange.start
|
|
686
833
|
? []
|
|
687
|
-
:
|
|
834
|
+
: sliceForTask(task, resolvedRange, task.maxWindowDays);
|
|
835
|
+
// 末端上限之外的窗整窗丢弃 → 与 runWindowed 同口径:计数可见 + 进度行(见那边的注释)。
|
|
836
|
+
const droppedByClamp = rawWindows.filter((w) => w.start >= limit);
|
|
837
|
+
const truncatedSource = rawWindows;
|
|
838
|
+
const clamped = clampWindowEnd(rawWindows, limit);
|
|
688
839
|
const skippedResidual = clamped.filter((w) => isResidualWindow(task, w));
|
|
689
|
-
const
|
|
840
|
+
const notResidual = clamped.filter((w) => !isResidualWindow(task, w));
|
|
841
|
+
const links = containedLinks(notResidual, overlapMs);
|
|
842
|
+
const windows = links.kept;
|
|
843
|
+
// 重叠压缩掉的窗:与 runWindowed 同口径(见那边的注释)—— 是否算「已覆盖」要等
|
|
844
|
+
// 覆盖窗的结果,由 runRotated 的结算段定(plan 里先带 links)。
|
|
845
|
+
const containedCount = 0;
|
|
690
846
|
const outRange = { start: rangeStart,
|
|
691
847
|
end: Math.max(rangeStart, Math.min(resolvedRange.end, limit)) };
|
|
692
848
|
const lock = { by: lockOwner(), at: new Date(now).toISOString() };
|
|
@@ -696,16 +852,28 @@ async function planRotation(task, ctx, now) {
|
|
|
696
852
|
if (windows.length === 0) {
|
|
697
853
|
for (const w of skippedResidual)
|
|
698
854
|
ctx.onProgress?.(residualNote(task, w));
|
|
699
|
-
|
|
855
|
+
for (const w of droppedByClamp)
|
|
856
|
+
ctx.onProgress?.(clampedAwayNote(task, w, limit));
|
|
857
|
+
for (const w of truncatedByClamp(truncatedSource, limit))
|
|
858
|
+
ctx.onProgress?.(clampedShortNote(task, w, limit));
|
|
859
|
+
return { task, state, resolvedRange: outRange, windows, skippedResidual, droppedByClamp, containedCount,
|
|
860
|
+
containedBy: links.containedBy, windowOk: [], truncatedSource, lock, gaps: [],
|
|
700
861
|
succeeded: 0, failed: 0, fetched: 0, written: 0,
|
|
701
862
|
contiguousEnd: null, contiguousBroken: false, resolvedEnd: null };
|
|
702
863
|
}
|
|
703
864
|
for (const w of skippedResidual)
|
|
704
865
|
ctx.onProgress?.(residualNote(task, w));
|
|
866
|
+
for (const w of droppedByClamp)
|
|
867
|
+
ctx.onProgress?.(clampedAwayNote(task, w, limit));
|
|
868
|
+
// 截短提示(Codex 2026-11-10 P2):考勤/绩效这类声明了 windowEndClamp 的任务在
|
|
869
|
+
// `sync run` 里**全部走轮转路径**,此前只有 runWindowed 出这句 —— 实际命令面看不到。
|
|
870
|
+
for (const w of truncatedByClamp(truncatedSource, limit))
|
|
871
|
+
ctx.onProgress?.(clampedShortNote(task, w, limit));
|
|
705
872
|
await withStoreRetries(ctx, () => ctx.store.transaction(async (tx) => {
|
|
706
873
|
await markPending(tx, task.id, state, lock);
|
|
707
874
|
}));
|
|
708
|
-
return { task, state, resolvedRange: outRange, windows, skippedResidual,
|
|
875
|
+
return { task, state, resolvedRange: outRange, windows, skippedResidual, droppedByClamp, containedCount,
|
|
876
|
+
containedBy: links.containedBy, windowOk: windows.map(() => false), truncatedSource, lock, gaps: [],
|
|
709
877
|
succeeded: 0, failed: 0, fetched: 0, written: 0,
|
|
710
878
|
contiguousEnd: null, contiguousBroken: false, resolvedEnd: null };
|
|
711
879
|
}
|
|
@@ -776,6 +944,7 @@ export async function runRotated(tasks, ctx) {
|
|
|
776
944
|
const written = await withRetries(ctx, () => commitWindow(ctx, task, w, kept, expected, watermarkRange, plan.lock), isTransient);
|
|
777
945
|
plan.written += written;
|
|
778
946
|
plan.succeeded++;
|
|
947
|
+
plan.windowOk[round] = true;
|
|
779
948
|
plan.resolvedEnd = watermarkMs;
|
|
780
949
|
if (!plan.contiguousBroken)
|
|
781
950
|
plan.contiguousEnd = w.end;
|
|
@@ -788,8 +957,10 @@ export async function runRotated(tasks, ctx) {
|
|
|
788
957
|
await storeGap(ctx, task, w, isMismatch ? 'count_mismatch' : 'write_failed', isMismatch ? e.message : failDetail('落库失败', e), plan.gaps);
|
|
789
958
|
continue;
|
|
790
959
|
}
|
|
791
|
-
|
|
792
|
-
|
|
960
|
+
for (const h of (Array.isArray(fetchRes.gapHint) ? fetchRes.gapHint : [fetchRes.gapHint])) {
|
|
961
|
+
if (h === undefined)
|
|
962
|
+
continue;
|
|
963
|
+
await storeGap(ctx, task, w, h.reason, gapHintDetail(h), plan.gaps);
|
|
793
964
|
}
|
|
794
965
|
progress(`[${task.id}] 轮转 round ${round + 1}/${maxRounds} 完成:拉取 ${fetchRes.rows.length} 行,写入 ${kept.length} 行`);
|
|
795
966
|
}
|
|
@@ -805,14 +976,35 @@ export async function runRotated(tasks, ctx) {
|
|
|
805
976
|
// 末端与水位同源(修复批 4 #2):断链时 = 连续成功末端(resolvedEnd),未断链 = 请求末。
|
|
806
977
|
// ⚠️ 末端是「上次成功推进到的时刻」,与删除边界(天对齐下取 dayOf(end−1ms),
|
|
807
978
|
// 见 writeRows)刻意不同:前者是位置、后者是行集覆盖。
|
|
808
|
-
return plans.map((p) =>
|
|
809
|
-
|
|
810
|
-
//
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
979
|
+
return plans.map((p) => {
|
|
980
|
+
// 合并窗结算(与 runWindowed 同口径,Codex 2026-11-10 第三轮 P2):覆盖窗成功才算
|
|
981
|
+
// contained;覆盖窗失败 → 该段没拉到 → 归 skipped 并逐窗出进度行。
|
|
982
|
+
let contained = 0;
|
|
983
|
+
let unreached = 0;
|
|
984
|
+
for (let i = 0; i < p.windows.length; i++) {
|
|
985
|
+
const merged = p.containedBy[i] ?? [];
|
|
986
|
+
if (merged.length === 0)
|
|
987
|
+
continue;
|
|
988
|
+
if (p.windowOk[i] === true)
|
|
989
|
+
contained += merged.length;
|
|
990
|
+
else {
|
|
991
|
+
unreached += merged.length;
|
|
992
|
+
for (const w of merged)
|
|
993
|
+
progress(containedUnreachedNote(p.task, w));
|
|
994
|
+
}
|
|
995
|
+
}
|
|
996
|
+
const skippedTotal = p.skippedResidual.length + p.droppedByClamp.length + unreached;
|
|
997
|
+
return {
|
|
998
|
+
taskId: p.task.id,
|
|
999
|
+
// 修复批 6 #2:残段窗计入 total、标 skipped(与 runWindowed 同口径)。
|
|
1000
|
+
windows: { total: p.windows.length + skippedTotal,
|
|
1001
|
+
succeeded: p.succeeded, failed: p.failed,
|
|
1002
|
+
skipped: skippedTotal,
|
|
1003
|
+
...(contained > 0 ? { contained } : {}) },
|
|
1004
|
+
rows: { fetched: p.fetched, written: p.written },
|
|
1005
|
+
resolvedRange: { start: p.resolvedRange.start, end: p.resolvedEnd ?? p.resolvedRange.start },
|
|
1006
|
+
gaps: p.gaps,
|
|
1007
|
+
status: p.failed === 0 ? 'complete' : 'partial',
|
|
1008
|
+
};
|
|
1009
|
+
});
|
|
818
1010
|
}
|
package/dist/sync/registry.js
CHANGED
|
@@ -3,6 +3,7 @@ import { REPORT_TASKS } from './tasks/reports.js';
|
|
|
3
3
|
import { TICKET_TASKS } from './tasks/tickets.js';
|
|
4
4
|
import { SESSIONS_TASKS } from './tasks/sessions.js';
|
|
5
5
|
import { KNOWLEDGE_TASKS } from './tasks/knowledge.js';
|
|
6
|
+
import { assertWriteContracts } from './write-contract.js';
|
|
6
7
|
/** 全量同步任务注册表:dim 6 + reports 9 + tickets 2 + sessions 1 + knowledge 3 = 21。 */
|
|
7
8
|
export const SYNC_TASKS = [
|
|
8
9
|
...DIM_TASKS,
|
|
@@ -11,6 +12,9 @@ export const SYNC_TASKS = [
|
|
|
11
12
|
...SESSIONS_TASKS,
|
|
12
13
|
...KNOWLEDGE_TASKS,
|
|
13
14
|
];
|
|
15
|
+
// 写策略声明前提的加载期校验(`write-contract.ts`):声明错了就在此炸,
|
|
16
|
+
// 而不是在某个真实租户的边界日里静默丢/留数据。
|
|
17
|
+
assertWriteContracts(SYNC_TASKS);
|
|
14
18
|
/** 按 id 取任务(sync run <id> 的分派入口)。 */
|
|
15
19
|
export function taskById(id) {
|
|
16
20
|
return SYNC_TASKS.find((t) => t.id === id);
|
package/dist/sync/state.js
CHANGED
|
@@ -1,3 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `sync_state.range_status` 的取值(闭集;`null` = 从未运行过)。
|
|
3
|
+
*
|
|
4
|
+
* ⚠️ 运行时数组 + `typeof ARR[number]` 而不是裸联合类型:这是**输出数据里的机器值**
|
|
5
|
+
* (`sync status` 的 `range_status`),Agent 要按它分支。值域必须能在运行时导出,
|
|
6
|
+
* `meta.field_notes` 的图例与测试才能双向核对「枚举 ⊆ 图例、图例 ⊆ 枚举」。
|
|
7
|
+
*
|
|
8
|
+
* ⚠️⚠️ **`complete` 的真实语义不是「全窗口成功」**(设计稿 §6.1 的原话),而是
|
|
9
|
+
* 「水位推进过」——引擎在**每个成功窗**的同一事务里调 `advanceWatermark`,它无条件写
|
|
10
|
+
* `complete`。2026-10-27 实测(3 窗、中窗失败):run 报 `status: partial`、
|
|
11
|
+
* `windows: {succeeded: 2, failed: 1}`,而 `range_status` 仍是 `complete`。
|
|
12
|
+
* 所以 `complete` **不能**用来判定「这段数据完整」;缺口要看 `gaps` / `sync_gaps`。
|
|
13
|
+
*
|
|
14
|
+
* 反过来 `pending` 也**不代表「一行都没写」**:首窗失败时 `contiguousEnd` 为 null,
|
|
15
|
+
* 后续成功窗的 `commitWindow` 拿到的 `watermarkRange` 是 null → 不调 `advanceWatermark`,
|
|
16
|
+
* 数据照常落库而状态停在 `markPending` 写的 `pending`。2026-10-27 实测:3 窗中首窗失败、
|
|
17
|
+
* 后两窗成功 → `range_status: pending`、`watermark: null`,但表里**有 2 行数据**。
|
|
18
|
+
* 所以 `pending` 只说明「水位没动过」,不说明「库里没数据」。
|
|
19
|
+
*/
|
|
20
|
+
export const RANGE_STATUSES = ['pending', 'complete'];
|
|
1
21
|
/** 读某 dataset 的水位状态行;不存在时全空。 */
|
|
2
22
|
export async function readWatermark(store, dataset) {
|
|
3
23
|
// 常量 SQL 调用点 1/3
|