pixivflow 2.19.5 → 2.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. package/dist/commands/SchedulerCommand.js +36 -0
  2. package/dist/commands/SchedulerRunOnceCommand.d.ts +2 -3
  3. package/dist/commands/SchedulerRunOnceCommand.js +2 -3
  4. package/dist/commands/scheduler-runtime.js +17 -1
  5. package/dist/config/types.d.ts +7 -0
  6. package/dist/config/validation.js +10 -0
  7. package/dist/delivery/DeliveryService.d.ts +21 -1
  8. package/dist/delivery/DeliveryService.js +2 -2
  9. package/dist/delivery/HttpMultipartDelivery.js +29 -8
  10. package/dist/delivery/OutboxWorker.d.ts +2 -0
  11. package/dist/delivery/OutboxWorker.js +3 -0
  12. package/dist/delivery/types.d.ts +18 -0
  13. package/dist/download/handlers/IllustrationTargetHandler.d.ts +0 -1
  14. package/dist/download/handlers/IllustrationTargetHandler.js +2 -13
  15. package/dist/download/handlers/NovelTargetHandler.js +2 -0
  16. package/dist/download/handlers/deliveryContext.d.ts +16 -0
  17. package/dist/download/handlers/deliveryContext.js +32 -0
  18. package/dist/notification/NotificationPolicy.d.ts +12 -0
  19. package/dist/notification/NotificationPolicy.js +86 -0
  20. package/dist/package.json +1 -1
  21. package/dist/scheduler/MultiScheduleManager.js +2 -0
  22. package/dist/scheduler/ScheduleTriggerServer.d.ts +58 -1
  23. package/dist/scheduler/ScheduleTriggerServer.js +207 -40
  24. package/dist/scheduler/SlotCoordinator.d.ts +80 -2
  25. package/dist/scheduler/SlotCoordinator.js +145 -3
  26. package/dist/storage/DatabaseMigration.js +11 -0
  27. package/dist/storage/repositories/SlotRepository.d.ts +8 -0
  28. package/dist/storage/repositories/SlotRepository.js +6 -2
  29. package/dist/version.js +1 -1
  30. package/dist/webui/package.json +1 -1
  31. package/package.json +1 -1
@@ -4,16 +4,89 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
4
4
  };
5
5
  Object.defineProperty(exports, "__esModule", { value: true });
6
6
  exports.ScheduleTriggerServer = void 0;
7
+ exports.scheduleAttemptId = scheduleAttemptId;
8
+ exports.scheduleProvider = scheduleProvider;
7
9
  const express_1 = __importDefault(require("express"));
8
10
  const node_crypto_1 = require("node:crypto");
9
11
  const logger_1 = require("../logger");
12
+ const TRIGGER_OUTCOMES = {
13
+ accepted: { event: 'schedule.trigger_accepted', httpStatus: 202, status: 'accepted', note: 'queued' },
14
+ already_running: {
15
+ event: 'schedule.trigger_already_running',
16
+ httpStatus: 202,
17
+ status: 'running',
18
+ note: 'already_running',
19
+ },
20
+ already_completed: {
21
+ event: 'schedule.trigger_already_completed',
22
+ httpStatus: 200,
23
+ status: 'completed',
24
+ note: 'already_completed',
25
+ },
26
+ rejected: { event: 'schedule.trigger_rejected', httpStatus: 503, status: 'rejected', note: 'rejected' },
27
+ };
28
+ /** Bounded, log-safe correlation token. */
29
+ const ATTEMPT_ID_MAX_LENGTH = 64;
30
+ const ATTEMPT_ID_UNSAFE = /[^A-Za-z0-9._:-]/g;
31
+ /**
32
+ * Correlation id issued by the calling clock (`x-schedule-attempt-id`, falling
33
+ * back to `x-attempt-id`).
34
+ *
35
+ * Purely diagnostic: it is trimmed, reduced to `[A-Za-z0-9._:-]` and capped, and
36
+ * it NEVER participates in occurrence identity. When the clock sends nothing
37
+ * usable the field is omitted rather than synthesized — a made-up id would make
38
+ * a missing clock indistinguishable from a satisfied one, which is exactly the
39
+ * blindness this logging exists to remove.
40
+ */
41
+ function scheduleAttemptId(req) {
42
+ const raw = req.headers['x-schedule-attempt-id'] ?? req.headers['x-attempt-id'];
43
+ const value = Array.isArray(raw) ? raw[0] : raw;
44
+ if (typeof value !== 'string')
45
+ return undefined;
46
+ const cleaned = value.trim().replace(ATTEMPT_ID_UNSAFE, '').slice(0, ATTEMPT_ID_MAX_LENGTH);
47
+ return cleaned.length > 0 ? cleaned : undefined;
48
+ }
49
+ /** Max length and character set of the provider tag, which is log-only metadata. */
50
+ const PROVIDER_MAX_LENGTH = 32;
51
+ const PROVIDER_UNSAFE = /[^A-Za-z0-9._-]/g;
52
+ /**
53
+ * Optional self-identification from the clock that fired: `X-Schedule-Provider`
54
+ * (e.g. `cron-job-org`, `cloudflare`, `manual`).
55
+ *
56
+ * OBSERVABILITY ONLY. It NEVER participates in authorization and NEVER in
57
+ * occurrence identity: a clock that lies about its name changes one log field
58
+ * and nothing else. That is deliberate — a self-declared header must not be able
59
+ * to alter which occurrence runs or whether the request is admitted. Sanitized
60
+ * and bounded like the attempt id, and omitted when the clock sends nothing
61
+ * usable rather than defaulted, so "we do not know which clock this was" stays
62
+ * distinguishable from "the clock identified itself".
63
+ */
64
+ function scheduleProvider(req) {
65
+ const raw = req.headers['x-schedule-provider'];
66
+ const value = Array.isArray(raw) ? raw[0] : raw;
67
+ if (typeof value !== 'string')
68
+ return undefined;
69
+ const cleaned = value.trim().replace(PROVIDER_UNSAFE, '').slice(0, PROVIDER_MAX_LENGTH);
70
+ return cleaned.length > 0 ? cleaned : undefined;
71
+ }
72
+ /** Epoch ms -> ISO 8601 UTC. Undefined when the value is not a usable instant. */
73
+ function isoUtc(epochMs) {
74
+ if (typeof epochMs !== 'number' || !Number.isFinite(epochMs))
75
+ return undefined;
76
+ const date = new Date(epochMs);
77
+ // `toISOString` throws on an out-of-range Date, and inside the handler that
78
+ // would turn a real 202 into a 500 — logging must never move the status code.
79
+ return Number.isNaN(date.getTime()) ? undefined : date.toISOString();
80
+ }
10
81
  class ScheduleTriggerServer {
11
82
  token;
12
83
  handlers;
84
+ refetchToken;
13
85
  server = null;
14
- constructor(token, handlers) {
86
+ constructor(token, handlers, refetchToken = process.env.PIXIVFLOW_REFETCH_TOKEN?.trim() || undefined) {
15
87
  this.token = token;
16
88
  this.handlers = handlers;
89
+ this.refetchToken = refetchToken;
17
90
  }
18
91
  /** Token from config or SCHEDULER_TRIGGER_TOKEN env; empty => fail closed. */
19
92
  static resolveToken(configured) {
@@ -22,6 +95,8 @@ class ScheduleTriggerServer {
22
95
  start(host, port) {
23
96
  const app = (0, express_1.default)();
24
97
  app.use(express_1.default.json());
98
+ // Correlation + clock start for every route. Cheap enough to be unconditional.
99
+ app.use(this.correlate);
25
100
  app.get('/health', (_req, res) => {
26
101
  res.json({ status: 'ok', service: 'pixivflow-scheduler-trigger' });
27
102
  });
@@ -32,11 +107,17 @@ class ScheduleTriggerServer {
32
107
  });
33
108
  // Trigger one schedule by id. The server resolves the occurrence from the
34
109
  // schedule cron; the body carries no date and cannot back-fill history.
35
- app.post('/internal/schedules/:scheduleId/run', this.auth, async (req, res) => {
110
+ //
111
+ // `received` runs BEFORE auth so a trigger that is never admitted (bad token,
112
+ // disabled endpoint) still leaves durable evidence that the clock fired —
113
+ // "the clock never arrived" and "the clock was rejected" used to look the
114
+ // same from outside: nothing at all.
115
+ app.post('/internal/schedules/:scheduleId/run', this.received, this.auth, async (req, res) => {
36
116
  try {
37
117
  const scheduleId = req.params.scheduleId;
38
118
  if (!this.handlers.listSchedules().includes(scheduleId)) {
39
119
  res.status(404).json({ status: 'error', error: `unknown schedule: ${scheduleId}` });
120
+ this.triggerOutcome('schedule.trigger_not_found', 404, req, res, { schedule_id: scheduleId });
40
121
  return;
41
122
  }
42
123
  // Optional human label for provenance (e.g. a deploy-layer "今日早班").
@@ -44,56 +125,79 @@ class ScheduleTriggerServer {
44
125
  const label = typeof req.body?.label === 'string' ? req.body.label.slice(0, 80) : undefined;
45
126
  const resolved = this.handlers.resolve(scheduleId, 'http', new Date(), label);
46
127
  if (!('context' in resolved)) {
128
+ // A resolve refusal (expired occurrence 410, too-early 425, bad cron
129
+ // 400) is a business rejection, so it reports `resolved.status`
130
+ // rather than the transport-level 503.
47
131
  res.status(resolved.status).json({ status: 'error', error: resolved.error });
132
+ this.triggerOutcome('schedule.trigger_rejected', resolved.status, req, res, {
133
+ schedule_id: scheduleId,
134
+ disposition: 'rejected',
135
+ reason: resolved.error,
136
+ });
48
137
  return;
49
138
  }
50
- const result = await this.handlers.run(scheduleId, resolved.context);
139
+ const context = resolved.context;
140
+ const result = await this.handlers.run(scheduleId, context);
51
141
  // Business disposition, not HTTP luck: an accepted or already-running
52
142
  // occurrence is NOT a completed one. Returning 200/"completed" for a run
53
143
  // that is still executing makes an external clock stop retrying and
54
144
  // silently lose the slot, so 'running' is reported as 202 here.
55
- switch (result.disposition) {
56
- case 'already_completed':
57
- res.status(200).json({
58
- status: 'completed',
59
- schedule: result,
60
- note: 'already_completed',
61
- });
62
- return;
63
- case 'accepted':
64
- res.status(202).json({
65
- status: 'accepted',
66
- schedule: result,
67
- note: 'queued',
68
- });
69
- return;
70
- case 'already_running':
71
- res.status(202).json({
72
- status: 'running',
73
- schedule: result,
74
- note: 'already_running',
75
- });
76
- return;
77
- default:
78
- res.status(503).json({
79
- status: 'rejected',
80
- schedule: result,
81
- note: 'rejected',
82
- });
83
- return;
84
- }
145
+ // Status code, body and log event all come from ONE table.
146
+ const spec = TRIGGER_OUTCOMES[result.disposition] ?? TRIGGER_OUTCOMES.rejected;
147
+ res.status(spec.httpStatus).json({ status: spec.status, schedule: result, note: spec.note });
148
+ this.triggerOutcome(spec.event, spec.httpStatus, req, res, {
149
+ schedule_id: scheduleId,
150
+ slot_id: context.slotId,
151
+ occurrence_at: isoUtc(context.occurrenceAt),
152
+ disposition: result.disposition,
153
+ reason: result.cells?.find((cell) => cell.error)?.error ?? undefined,
154
+ trigger_source: context.triggerSource,
155
+ });
85
156
  }
86
157
  catch (error) {
87
158
  logger_1.logger.error('Schedule trigger failed', { error: error instanceof Error ? error.message : String(error) });
88
159
  // The slot ledger resumes on the next trigger; a 500 tells the clock to
89
160
  // retry safely (idempotent — the same occurrence/ slot is reused).
90
161
  res.status(500).json({ status: 'error', error: 'schedule run failed; the occurrence will resume on the next trigger' });
162
+ this.triggerOutcome('schedule.trigger_error', 500, req, res, {
163
+ schedule_id: req.params.scheduleId,
164
+ reason: error instanceof Error ? error.message : String(error),
165
+ });
166
+ }
167
+ });
168
+ app.post('/internal/targets/:targetId/refetch', this.refetchAuth, async (req, res) => {
169
+ const requestId = req.body?.requestId;
170
+ if (typeof requestId !== 'string' || !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(requestId)) {
171
+ res.status(400).json({ status: 'error', error: 'requestId must be a UUID' });
172
+ return;
173
+ }
174
+ // Opaque caller correlation (review chain / review id). Optional; bounded
175
+ // length, never interpreted here. Recorded with the manual Slot so a
176
+ // recovered worker still correlates the outcome with the requester.
177
+ const correlationId = req.body?.correlationId;
178
+ if (correlationId !== undefined && (typeof correlationId !== 'string' || correlationId.length > 200)) {
179
+ res.status(400).json({ status: 'error', error: 'correlationId must be a string of at most 200 chars' });
180
+ return;
181
+ }
182
+ if (!this.handlers.refetch) {
183
+ res.status(503).json({ status: 'error', error: 'refetch is unavailable' });
184
+ return;
185
+ }
186
+ try {
187
+ const result = await this.handlers.refetch(req.params.targetId, requestId, correlationId ?? undefined);
188
+ res.status(202).json({ status: 'accepted', ...result });
189
+ }
190
+ catch (error) {
191
+ const message = error instanceof Error ? error.message : String(error);
192
+ const status = message === 'unknown target' ? 404 : message === 'ambiguous target' ? 409 : 500;
193
+ logger_1.logger.warn('Manual refetch rejected', { targetId: req.params.targetId, requestId, status, error: message });
194
+ res.status(status).json({ status: 'error', error: status === 500 ? 'refetch admission failed' : message });
91
195
  }
92
196
  });
93
197
  // Convergence endpoint: after a machine stop/start, an operator or an
94
198
  // external watcher can ask the process to flush due deliveries/notifications
95
199
  // without running candidate selection. Deployment-agnostic (no platform refs).
96
- app.post('/internal/outbox/drain', this.auth, async (_req, res) => {
200
+ app.post('/internal/outbox/drain', this.auth, async (req, res) => {
97
201
  try {
98
202
  if (!this.handlers.drainOutbox) {
99
203
  res.status(503).json({ status: 'error', error: 'outbox worker not available in this runtime' });
@@ -103,7 +207,11 @@ class ScheduleTriggerServer {
103
207
  res.json({ status: 'ok', result });
104
208
  }
105
209
  catch (error) {
106
- logger_1.logger.error('Outbox drain failed', { error: error instanceof Error ? error.message : String(error) });
210
+ // Same correlation as the trigger lines, at zero extra cost.
211
+ logger_1.logger.error('Outbox drain failed', {
212
+ ...this.attemptMeta(req, res),
213
+ error: error instanceof Error ? error.message : String(error),
214
+ });
107
215
  res.status(500).json({ status: 'error', error: 'outbox drain failed; rows remain durable and retry' });
108
216
  }
109
217
  });
@@ -111,18 +219,76 @@ class ScheduleTriggerServer {
111
219
  logger_1.logger.info('Schedule trigger server listening', { host, port, auth: this.token ? 'bearer' : 'DISABLED (no token)' });
112
220
  });
113
221
  }
222
+ /**
223
+ * Per-request correlation, installed before every route. It records only the
224
+ * sanitized attempt id and the arrival instant, so each later line can report
225
+ * the same `attempt_id` and a handler-relative `elapsed_ms`.
226
+ */
227
+ correlate = (req, res, next) => {
228
+ res.locals.triggerStartedAt = Date.now();
229
+ res.locals.triggerAttemptId = scheduleAttemptId(req);
230
+ res.locals.triggerProvider = scheduleProvider(req);
231
+ next();
232
+ };
233
+ /**
234
+ * `trigger.received` — the clock arrived. Runs before auth, so it carries only
235
+ * non-sensitive request metadata: never the Authorization header, the bearer
236
+ * value, the configured token, or any fragment of them.
237
+ */
238
+ received = (req, res, next) => {
239
+ logger_1.logger.info('Schedule trigger received', { event: 'schedule.trigger_received', ...this.attemptMeta(req, res) });
240
+ next();
241
+ };
242
+ /**
243
+ * The single writer for every post-resolution `trigger.*` line. `event` and
244
+ * `http_status` are passed in from the caller's outcome table, so the line
245
+ * always describes the same status the response carried.
246
+ */
247
+ triggerOutcome(event, httpStatus, req, res, extra) {
248
+ logger_1.logger.info('Schedule trigger outcome', {
249
+ event,
250
+ ...this.attemptMeta(req, res),
251
+ http_status: httpStatus,
252
+ ...extra,
253
+ });
254
+ }
255
+ /**
256
+ * The non-sensitive correlation fields shared by every `schedule.trigger_*`
257
+ * line. `attempt_id` and `provider` are OMITTED (never defaulted) when the
258
+ * clock did not send them, so an unidentified clock stays visible as such.
259
+ */
260
+ attemptMeta(req, res) {
261
+ const attemptId = res.locals.triggerAttemptId;
262
+ const provider = res.locals.triggerProvider;
263
+ const startedAt = res.locals.triggerStartedAt ?? Date.now();
264
+ return {
265
+ ...(attemptId ? { attempt_id: attemptId } : {}),
266
+ ...(provider ? { provider } : {}),
267
+ path: req.path,
268
+ method: req.method,
269
+ elapsed_ms: Date.now() - startedAt,
270
+ };
271
+ }
114
272
  auth = (req, res, next) => {
115
- if (!this.token) {
116
- // Fail closed: never allow an unauthenticated trigger in production.
117
- res.status(503).json({ status: 'error', error: 'trigger disabled: SCHEDULER_TRIGGER_TOKEN not configured' });
273
+ this.authenticate(this.token, req, res, next);
274
+ };
275
+ refetchAuth = (req, res, next) => {
276
+ this.authenticate(this.refetchToken, req, res, next);
277
+ };
278
+ authenticate(expected, req, res, next) {
279
+ if (!expected) {
280
+ // Fail closed: never allow an unauthenticated trigger in production. This
281
+ // is an admission failure, so it is reported with the status actually sent.
282
+ res.status(503).json({ status: 'error', error: 'endpoint disabled: token not configured' });
283
+ this.triggerOutcome('schedule.trigger_unauthorized', 503, req, res, { reason: 'endpoint disabled: no token configured' });
118
284
  return;
119
285
  }
120
286
  const header = req.headers.authorization ?? '';
121
287
  const presented = header.startsWith('Bearer ') ? header.slice(7).trim() : '';
122
- const expected = this.token;
123
288
  let ok = presented.length === expected.length;
124
289
  if (ok) {
125
290
  try {
291
+ // Values only, never logged: the timing check is deliberately opaque.
126
292
  ok = (0, node_crypto_1.timingSafeEqual)(Buffer.from(presented), Buffer.from(expected));
127
293
  }
128
294
  catch {
@@ -131,10 +297,11 @@ class ScheduleTriggerServer {
131
297
  }
132
298
  if (!ok) {
133
299
  res.status(401).json({ status: 'error', error: 'unauthorized' });
300
+ this.triggerOutcome('schedule.trigger_unauthorized', 401, req, res, { reason: 'invalid or missing bearer token' });
134
301
  return;
135
302
  }
136
303
  next();
137
- };
304
+ }
138
305
  stop() {
139
306
  this.server?.close();
140
307
  this.server = null;
@@ -18,8 +18,8 @@ import { TargetExecutionContext, WorkBinding } from './WorkIdentity';
18
18
  export declare const SLOT_LEASE_TTL_MS: number;
19
19
  export declare const SLOT_HEARTBEAT_MS: number;
20
20
  /**
21
- * Durable execution context attached to a run. A scheduled occurrence always
22
- * has a slotId; an ad-hoc/manual run has none (it never touches the slot ledger).
21
+ * Durable execution context attached to a scheduled occurrence or a remote
22
+ * manual replacement. The local run-once CLI has no SlotContext.
23
23
  */
24
24
  export interface SlotContext {
25
25
  slotId: string;
@@ -36,6 +36,16 @@ export interface SlotContext {
36
36
  */
37
37
  slotName: string;
38
38
  slotDate: string;
39
+ /**
40
+ * Remote manual replacement ("重抓") request UUID. Present only on slots
41
+ * opened by the authenticated refetch endpoint; null for scheduled
42
+ * occurrences. Persisted with the slot so a sleeping worker that recovers
43
+ * the slot still knows it was a manual replacement (delivery correlation,
44
+ * outcome reporting) without any in-memory state.
45
+ */
46
+ manualRequestId?: string;
47
+ /** Opaque caller correlation (review chain / review id). Null unless manual. */
48
+ correlationId?: string;
39
49
  }
40
50
  export interface SlotCellSummary {
41
51
  targetId: string;
@@ -50,6 +60,54 @@ export interface SlotRunSummary {
50
60
  alreadyCompleted: boolean;
51
61
  cells: SlotCellSummary[];
52
62
  }
63
+ /** Terminal statuses only: the rollup that ends an occurrence's life. */
64
+ export type ScheduleOutcomeStatus = 'success' | 'partial' | 'failed';
65
+ export interface ScheduleOutcomeTarget {
66
+ target_id: string;
67
+ status: CellStatus;
68
+ work_id: string | null;
69
+ error: string | null;
70
+ }
71
+ /**
72
+ * Business categories an operator actually asks about, counted per cell. The
73
+ * point is to separate "nothing suitable existed" (no_match) and "already
74
+ * delivered" (duplicate) from real failures: lumping them together made a
75
+ * healthy run look broken, and a broken one look routine.
76
+ *
77
+ * Each cell lands in exactly ONE category, in this precedence:
78
+ * submitted -> no_match -> duplicate -> delivery_failed -> executor_failed.
79
+ * `delivery_failed` is checked before `executor_failed` because a terminally
80
+ * lost delivery ALSO leaves the cell in the `failed` state — counting it twice
81
+ * would invent a second failure that does not exist.
82
+ */
83
+ export interface ScheduleOutcomeCells {
84
+ total: number;
85
+ submitted: number;
86
+ no_match: number;
87
+ duplicate: number;
88
+ /** True when there is at least one non-submitted cell and ALL of them are duplicates. */
89
+ all_duplicates: boolean;
90
+ executor_failed: number;
91
+ delivery_failed: number;
92
+ targets: ScheduleOutcomeTarget[];
93
+ }
94
+ /**
95
+ * The single structured terminal record for one schedule occurrence. The SAME
96
+ * object is both logged as `schedule.outcome` and persisted into the existing
97
+ * `execution.summary` detail, so the log line and the durable row cannot
98
+ * disagree — and no second event name or shadow ledger is introduced.
99
+ */
100
+ export interface ScheduleOutcomeRecord {
101
+ event: 'schedule.outcome';
102
+ schedule_id: string;
103
+ slot_id: string;
104
+ occurrence_at: string | undefined;
105
+ occurrence_date: string;
106
+ status: ScheduleOutcomeStatus;
107
+ /** From the slot row's own started_at/completed_at columns, when both exist. */
108
+ duration_ms: number | undefined;
109
+ cells: ScheduleOutcomeCells;
110
+ }
53
111
  export interface SlotResolveResult {
54
112
  context?: SlotContext;
55
113
  error?: string;
@@ -196,11 +254,31 @@ export declare class SlotCoordinator {
196
254
  releaseRunLease(slotId: string, owner: string): void;
197
255
  /** Roll cell results up into the slot status (one place computes the aggregate). */
198
256
  finish(slot: SlotContext, schedule: ScheduleConfig, targets: TargetConfig[]): SlotRunSummary;
257
+ /**
258
+ * Project the terminal slot row into the one structured outcome record. Every
259
+ * field comes from durable state — the slot row, its cells, and the delivery
260
+ * ledger. Nothing is inferred from error-message text.
261
+ */
262
+ private scheduleOutcome;
263
+ /**
264
+ * One cell -> one business category. `delivery_failed` is derived from the
265
+ * durable delivery ledger through the SAME port the FSM already uses, never
266
+ * from an error string, and it is decided BEFORE `executor_failed` because a
267
+ * terminally lost delivery also leaves the cell `failed`: one cause, one count.
268
+ */
269
+ private classifyCell;
199
270
  /**
200
271
  * Persist a one-row incident/execution summary into delivery_events
201
272
  * (event='execution.summary') at the terminal rollup only. Event-row storage
202
273
  * reuses the audit table + `runs show` read path instead of inventing a
203
274
  * summary table or overloading execution_log's illustration/novel typing.
275
+ *
276
+ * The `schedule.outcome` line is emitted HERE, immediately after that durable
277
+ * write and behind the SAME `hasExecutionSummary` dedupe, because the summary
278
+ * row is the occurrence-level terminal identity that already exists. Recovery
279
+ * that re-rolls the same terminal slot therefore logs nothing a second time,
280
+ * with no shadow ledger and no second event name. The identical object goes to
281
+ * both sinks, so the line and the row cannot disagree.
204
282
  */
205
283
  private persistExecutionSummary;
206
284
  completedSummary(slotId: string, schedule: ScheduleConfig): SlotRunSummary;
@@ -25,6 +25,31 @@ function deliveryTargetOf(target) {
25
25
  const name = target.delivery?.target;
26
26
  return typeof name === 'string' && name.trim() ? name.trim() : null;
27
27
  }
28
+ /**
29
+ * SQLite `CURRENT_TIMESTAMP` is UTC "YYYY-MM-DD HH:MM:SS". It carries no zone
30
+ * marker, and `Date.parse` reads that shape as LOCAL time — so the zone is
31
+ * added explicitly instead of being trusted to the engine.
32
+ */
33
+ function sqliteUtcMs(value) {
34
+ if (!value)
35
+ return undefined;
36
+ const normalized = /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/.test(value)
37
+ ? `${value.replace(' ', 'T')}Z`
38
+ : value;
39
+ const ms = Date.parse(normalized);
40
+ return Number.isNaN(ms) ? undefined : ms;
41
+ }
42
+ /**
43
+ * Epoch ms -> ISO 8601 UTC, or undefined when it is not a usable instant.
44
+ * `toISOString` throws on an out-of-range Date, and a throw here would abort the
45
+ * terminal rollup — a logging field must never be able to break dispatch.
46
+ */
47
+ function isoUtcOrUndefined(epochMs) {
48
+ if (typeof epochMs !== 'number' || !Number.isFinite(epochMs))
49
+ return undefined;
50
+ const date = new Date(epochMs);
51
+ return Number.isNaN(date.getTime()) ? undefined : date.toISOString();
52
+ }
28
53
  /**
29
54
  * Owns the Schedule Slot ledger for one scheduler run. A Slot is one durable
30
55
  * execution occurrence of a Schedule (NOT a morning/evening row). It ensures
@@ -102,6 +127,8 @@ class SlotCoordinator {
102
127
  triggerSource: slot.triggerSource,
103
128
  slotDate: slot.slotDate,
104
129
  slotName: slot.slotName,
130
+ manualRequestId: slot.manualRequestId ?? null,
131
+ correlationId: slot.correlationId ?? null,
105
132
  });
106
133
  if (created) {
107
134
  // Freeze membership: materialize one cell per target id from the snapshot.
@@ -347,25 +374,139 @@ class SlotCoordinator {
347
374
  }
348
375
  const status = this.database.slots.deriveSlotStatus(slot.slotId);
349
376
  this.database.slots.markSlotStatus(slot.slotId, status);
350
- this.persistExecutionSummary(slot.slotId, status);
351
377
  const cells = this.database.slots.getCells(slot.slotId).map((c) => ({
352
378
  targetId: c.targetId,
353
379
  status: c.status,
354
380
  workId: c.workId,
355
381
  error: c.lastError,
356
382
  }));
383
+ // Rolled up AFTER the slot row carries its terminal status/completed_at, so
384
+ // the outcome reports the durable timestamps rather than a fresh clock read.
385
+ this.persistExecutionSummary(slot.slotId, status, this.scheduleOutcome(slot, status, targets));
357
386
  const icon = (s) => s === 'submitted' ? '✅' : s === 'no_candidate' ? '⚠️ no_candidate' : '❌ failed';
358
387
  logger_1.logger.info(`Slot ${slot.slotId} ${status}\n` +
359
388
  cells.map((c) => ` ${c.targetId.padEnd(28)} ${icon(c.status)} ${c.workId ? '#' + c.workId : ''} ${c.error ? '(' + c.error + ')' : ''}`).join('\n'), { slot: slot.slotId, status });
360
389
  return { scheduleId: schedule.id, slotId: slot.slotId, status, alreadyCompleted: false, cells };
361
390
  }
391
+ /**
392
+ * Project the terminal slot row into the one structured outcome record. Every
393
+ * field comes from durable state — the slot row, its cells, and the delivery
394
+ * ledger. Nothing is inferred from error-message text.
395
+ */
396
+ scheduleOutcome(slot, status, targets) {
397
+ const slotRec = this.database.slots.getSlot(slot.slotId);
398
+ const cellRows = this.database.slots.getCells(slot.slotId);
399
+ // Only targets the caller still knows about can name a delivery channel; a
400
+ // cell with no known channel has no delivery fact to consult.
401
+ const deliveryTargetByTargetId = new Map();
402
+ for (const target of targets) {
403
+ const deliveryTarget = deliveryTargetOf(target);
404
+ if (target.id && deliveryTarget)
405
+ deliveryTargetByTargetId.set(target.id, deliveryTarget);
406
+ }
407
+ let submitted = 0;
408
+ let no_match = 0;
409
+ let duplicate = 0;
410
+ let executor_failed = 0;
411
+ let delivery_failed = 0;
412
+ const targetsDetail = cellRows.map((cell) => {
413
+ const classified = this.classifyCell(cell.status, slot.slotId, cell.targetId, deliveryTargetByTargetId);
414
+ if (classified === 'submitted')
415
+ submitted += 1;
416
+ else if (classified === 'no_match')
417
+ no_match += 1;
418
+ else if (classified === 'duplicate')
419
+ duplicate += 1;
420
+ else if (classified === 'delivery_failed')
421
+ delivery_failed += 1;
422
+ else if (classified === 'executor_failed')
423
+ executor_failed += 1;
424
+ return {
425
+ target_id: cell.targetId,
426
+ status: cell.status,
427
+ work_id: cell.workId,
428
+ error: cell.lastError,
429
+ };
430
+ });
431
+ // A fully-submitted slot has no non-submitted cell at all, and reporting
432
+ // `all_duplicates` there would be a lie — so the count must be non-zero.
433
+ const nonSubmitted = cellRows.length - submitted;
434
+ const all_duplicates = nonSubmitted > 0 && duplicate === nonSubmitted;
435
+ const startedMs = sqliteUtcMs(slotRec?.startedAt);
436
+ const completedMs = sqliteUtcMs(slotRec?.completedAt);
437
+ const duration_ms = startedMs !== undefined && completedMs !== undefined
438
+ ? Math.max(0, completedMs - startedMs)
439
+ : undefined;
440
+ const occurrenceAt = slotRec?.occurrenceAt ?? slot.occurrenceAt;
441
+ return {
442
+ event: 'schedule.outcome',
443
+ schedule_id: slotRec?.scheduleId ?? slot.scheduleId,
444
+ slot_id: slot.slotId,
445
+ occurrence_at: isoUtcOrUndefined(occurrenceAt),
446
+ occurrence_date: slotRec?.occurrenceDate ?? slot.occurrenceDate,
447
+ status: status,
448
+ duration_ms,
449
+ cells: {
450
+ total: cellRows.length,
451
+ submitted,
452
+ no_match,
453
+ duplicate,
454
+ all_duplicates,
455
+ executor_failed,
456
+ delivery_failed,
457
+ targets: targetsDetail,
458
+ },
459
+ };
460
+ }
461
+ /**
462
+ * One cell -> one business category. `delivery_failed` is derived from the
463
+ * durable delivery ledger through the SAME port the FSM already uses, never
464
+ * from an error string, and it is decided BEFORE `executor_failed` because a
465
+ * terminally lost delivery also leaves the cell `failed`: one cause, one count.
466
+ */
467
+ classifyCell(cellStatus, slotId, targetId, deliveryTargetByTargetId) {
468
+ if (cellStatus === 'submitted')
469
+ return 'submitted';
470
+ if (cellStatus === 'no_candidate')
471
+ return 'no_match';
472
+ if (cellStatus === 'duplicate')
473
+ return 'duplicate';
474
+ const deliveryTarget = deliveryTargetByTargetId.get(targetId);
475
+ if (deliveryTarget && this.delivery) {
476
+ try {
477
+ const state = this.delivery.stateFor({ deliveryTarget, slotId, targetId });
478
+ if (state.kind === 'lost')
479
+ return 'delivery_failed';
480
+ }
481
+ catch (error) {
482
+ // Observability must never break the rollup.
483
+ logger_1.logger.debug('Delivery state unavailable for outcome rollup', {
484
+ slot: slotId,
485
+ target: targetId,
486
+ error: error.message,
487
+ });
488
+ }
489
+ }
490
+ if (cellStatus === 'failed')
491
+ return 'executor_failed';
492
+ // Non-terminal (delivery_pending / artifact_ready): such a slot is not
493
+ // terminal either, so no outcome record is written for it at all.
494
+ return 'other';
495
+ }
362
496
  /**
363
497
  * Persist a one-row incident/execution summary into delivery_events
364
498
  * (event='execution.summary') at the terminal rollup only. Event-row storage
365
499
  * reuses the audit table + `runs show` read path instead of inventing a
366
500
  * summary table or overloading execution_log's illustration/novel typing.
501
+ *
502
+ * The `schedule.outcome` line is emitted HERE, immediately after that durable
503
+ * write and behind the SAME `hasExecutionSummary` dedupe, because the summary
504
+ * row is the occurrence-level terminal identity that already exists. Recovery
505
+ * that re-rolls the same terminal slot therefore logs nothing a second time,
506
+ * with no shadow ledger and no second event name. The identical object goes to
507
+ * both sinks, so the line and the row cannot disagree.
367
508
  */
368
- persistExecutionSummary(slotId, status) {
509
+ persistExecutionSummary(slotId, status, outcome) {
369
510
  if (status !== 'success' && status !== 'partial' && status !== 'failed')
370
511
  return;
371
512
  try {
@@ -377,8 +518,9 @@ class SlotCoordinator {
377
518
  slotId,
378
519
  event: 'execution.summary',
379
520
  countsAsAttempt: 0,
380
- detail: { summary },
521
+ detail: { summary, outcome },
381
522
  });
523
+ logger_1.logger.info('Schedule occurrence reached a terminal outcome', outcome);
382
524
  }
383
525
  catch (error) {
384
526
  logger_1.logger.debug('Failed to persist execution summary', { slot: slotId, error: error.message });
@@ -99,6 +99,15 @@ class DatabaseMigration {
99
99
  lease_owner TEXT,
100
100
  lease_until INTEGER,
101
101
  heartbeat_at INTEGER,
102
+ -- Request UUID of a remote manual replacement ("重抓"). NULL for a
103
+ -- scheduled occurrence. Persisted in the SAME transaction that
104
+ -- opens the manual slot, so the acceptance ACK and the caller's
105
+ -- retry converge on one row instead of racing a second slot.
106
+ manual_request_id TEXT,
107
+ -- Opaque caller correlation (review chain / review id). Recorded so
108
+ -- the terminal outcome can be reported back to the requester
109
+ -- without this service learning anything about the review.
110
+ correlation_id TEXT,
102
111
  created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
103
112
  started_at DATETIME,
104
113
  completed_at DATETIME,
@@ -227,6 +236,8 @@ class DatabaseMigration {
227
236
  lease_owner: 'ALTER TABLE schedule_slots ADD COLUMN lease_owner TEXT',
228
237
  lease_until: 'ALTER TABLE schedule_slots ADD COLUMN lease_until INTEGER',
229
238
  heartbeat_at: 'ALTER TABLE schedule_slots ADD COLUMN heartbeat_at INTEGER',
239
+ manual_request_id: 'ALTER TABLE schedule_slots ADD COLUMN manual_request_id TEXT',
240
+ correlation_id: 'ALTER TABLE schedule_slots ADD COLUMN correlation_id TEXT',
230
241
  };
231
242
  const columnAlters = [];
232
243
  for (const [col, sql] of Object.entries(slotColumnMigrations)) {