pixivflow 2.19.5 → 2.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/SchedulerCommand.js +36 -0
- package/dist/commands/SchedulerRunOnceCommand.d.ts +2 -3
- package/dist/commands/SchedulerRunOnceCommand.js +2 -3
- package/dist/commands/scheduler-runtime.js +17 -1
- package/dist/config/types.d.ts +7 -0
- package/dist/config/validation.js +10 -0
- package/dist/delivery/DeliveryService.d.ts +21 -1
- package/dist/delivery/DeliveryService.js +2 -2
- package/dist/delivery/HttpMultipartDelivery.js +29 -8
- package/dist/delivery/OutboxWorker.d.ts +2 -0
- package/dist/delivery/OutboxWorker.js +3 -0
- package/dist/delivery/types.d.ts +18 -0
- package/dist/download/handlers/IllustrationTargetHandler.d.ts +0 -1
- package/dist/download/handlers/IllustrationTargetHandler.js +2 -13
- package/dist/download/handlers/NovelTargetHandler.js +2 -0
- package/dist/download/handlers/deliveryContext.d.ts +16 -0
- package/dist/download/handlers/deliveryContext.js +32 -0
- package/dist/notification/NotificationPolicy.d.ts +12 -0
- package/dist/notification/NotificationPolicy.js +86 -0
- package/dist/package.json +1 -1
- package/dist/scheduler/MultiScheduleManager.js +2 -0
- package/dist/scheduler/ScheduleTriggerServer.d.ts +58 -1
- package/dist/scheduler/ScheduleTriggerServer.js +207 -40
- package/dist/scheduler/SlotCoordinator.d.ts +80 -2
- package/dist/scheduler/SlotCoordinator.js +145 -3
- package/dist/storage/DatabaseMigration.js +11 -0
- package/dist/storage/repositories/SlotRepository.d.ts +8 -0
- package/dist/storage/repositories/SlotRepository.js +6 -2
- package/dist/version.js +1 -1
- package/dist/webui/package.json +1 -1
- package/package.json +1 -1
|
@@ -4,16 +4,89 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.ScheduleTriggerServer = void 0;
|
|
7
|
+
exports.scheduleAttemptId = scheduleAttemptId;
|
|
8
|
+
exports.scheduleProvider = scheduleProvider;
|
|
7
9
|
const express_1 = __importDefault(require("express"));
|
|
8
10
|
const node_crypto_1 = require("node:crypto");
|
|
9
11
|
const logger_1 = require("../logger");
|
|
12
|
+
const TRIGGER_OUTCOMES = {
|
|
13
|
+
accepted: { event: 'schedule.trigger_accepted', httpStatus: 202, status: 'accepted', note: 'queued' },
|
|
14
|
+
already_running: {
|
|
15
|
+
event: 'schedule.trigger_already_running',
|
|
16
|
+
httpStatus: 202,
|
|
17
|
+
status: 'running',
|
|
18
|
+
note: 'already_running',
|
|
19
|
+
},
|
|
20
|
+
already_completed: {
|
|
21
|
+
event: 'schedule.trigger_already_completed',
|
|
22
|
+
httpStatus: 200,
|
|
23
|
+
status: 'completed',
|
|
24
|
+
note: 'already_completed',
|
|
25
|
+
},
|
|
26
|
+
rejected: { event: 'schedule.trigger_rejected', httpStatus: 503, status: 'rejected', note: 'rejected' },
|
|
27
|
+
};
|
|
28
|
+
/** Bounded, log-safe correlation token. */
|
|
29
|
+
const ATTEMPT_ID_MAX_LENGTH = 64;
|
|
30
|
+
const ATTEMPT_ID_UNSAFE = /[^A-Za-z0-9._:-]/g;
|
|
31
|
+
/**
|
|
32
|
+
* Correlation id issued by the calling clock (`x-schedule-attempt-id`, falling
|
|
33
|
+
* back to `x-attempt-id`).
|
|
34
|
+
*
|
|
35
|
+
* Purely diagnostic: it is trimmed, reduced to `[A-Za-z0-9._:-]` and capped, and
|
|
36
|
+
* it NEVER participates in occurrence identity. When the clock sends nothing
|
|
37
|
+
* usable the field is omitted rather than synthesized — a made-up id would make
|
|
38
|
+
* a missing clock indistinguishable from a satisfied one, which is exactly the
|
|
39
|
+
* blindness this logging exists to remove.
|
|
40
|
+
*/
|
|
41
|
+
function scheduleAttemptId(req) {
|
|
42
|
+
const raw = req.headers['x-schedule-attempt-id'] ?? req.headers['x-attempt-id'];
|
|
43
|
+
const value = Array.isArray(raw) ? raw[0] : raw;
|
|
44
|
+
if (typeof value !== 'string')
|
|
45
|
+
return undefined;
|
|
46
|
+
const cleaned = value.trim().replace(ATTEMPT_ID_UNSAFE, '').slice(0, ATTEMPT_ID_MAX_LENGTH);
|
|
47
|
+
return cleaned.length > 0 ? cleaned : undefined;
|
|
48
|
+
}
|
|
49
|
+
/** Max length and character set of the provider tag, which is log-only metadata. */
|
|
50
|
+
const PROVIDER_MAX_LENGTH = 32;
|
|
51
|
+
const PROVIDER_UNSAFE = /[^A-Za-z0-9._-]/g;
|
|
52
|
+
/**
|
|
53
|
+
* Optional self-identification from the clock that fired: `X-Schedule-Provider`
|
|
54
|
+
* (e.g. `cron-job-org`, `cloudflare`, `manual`).
|
|
55
|
+
*
|
|
56
|
+
* OBSERVABILITY ONLY. It NEVER participates in authorization and NEVER in
|
|
57
|
+
* occurrence identity: a clock that lies about its name changes one log field
|
|
58
|
+
* and nothing else. That is deliberate — a self-declared header must not be able
|
|
59
|
+
* to alter which occurrence runs or whether the request is admitted. Sanitized
|
|
60
|
+
* and bounded like the attempt id, and omitted when the clock sends nothing
|
|
61
|
+
* usable rather than defaulted, so "we do not know which clock this was" stays
|
|
62
|
+
* distinguishable from "the clock identified itself".
|
|
63
|
+
*/
|
|
64
|
+
function scheduleProvider(req) {
|
|
65
|
+
const raw = req.headers['x-schedule-provider'];
|
|
66
|
+
const value = Array.isArray(raw) ? raw[0] : raw;
|
|
67
|
+
if (typeof value !== 'string')
|
|
68
|
+
return undefined;
|
|
69
|
+
const cleaned = value.trim().replace(PROVIDER_UNSAFE, '').slice(0, PROVIDER_MAX_LENGTH);
|
|
70
|
+
return cleaned.length > 0 ? cleaned : undefined;
|
|
71
|
+
}
|
|
72
|
+
/** Epoch ms -> ISO 8601 UTC. Undefined when the value is not a usable instant. */
|
|
73
|
+
function isoUtc(epochMs) {
|
|
74
|
+
if (typeof epochMs !== 'number' || !Number.isFinite(epochMs))
|
|
75
|
+
return undefined;
|
|
76
|
+
const date = new Date(epochMs);
|
|
77
|
+
// `toISOString` throws on an out-of-range Date, and inside the handler that
|
|
78
|
+
// would turn a real 202 into a 500 — logging must never move the status code.
|
|
79
|
+
return Number.isNaN(date.getTime()) ? undefined : date.toISOString();
|
|
80
|
+
}
|
|
10
81
|
class ScheduleTriggerServer {
|
|
11
82
|
token;
|
|
12
83
|
handlers;
|
|
84
|
+
refetchToken;
|
|
13
85
|
server = null;
|
|
14
|
-
constructor(token, handlers) {
|
|
86
|
+
constructor(token, handlers, refetchToken = process.env.PIXIVFLOW_REFETCH_TOKEN?.trim() || undefined) {
|
|
15
87
|
this.token = token;
|
|
16
88
|
this.handlers = handlers;
|
|
89
|
+
this.refetchToken = refetchToken;
|
|
17
90
|
}
|
|
18
91
|
/** Token from config or SCHEDULER_TRIGGER_TOKEN env; empty => fail closed. */
|
|
19
92
|
static resolveToken(configured) {
|
|
@@ -22,6 +95,8 @@ class ScheduleTriggerServer {
|
|
|
22
95
|
start(host, port) {
|
|
23
96
|
const app = (0, express_1.default)();
|
|
24
97
|
app.use(express_1.default.json());
|
|
98
|
+
// Correlation + clock start for every route. Cheap enough to be unconditional.
|
|
99
|
+
app.use(this.correlate);
|
|
25
100
|
app.get('/health', (_req, res) => {
|
|
26
101
|
res.json({ status: 'ok', service: 'pixivflow-scheduler-trigger' });
|
|
27
102
|
});
|
|
@@ -32,11 +107,17 @@ class ScheduleTriggerServer {
|
|
|
32
107
|
});
|
|
33
108
|
// Trigger one schedule by id. The server resolves the occurrence from the
|
|
34
109
|
// schedule cron; the body carries no date and cannot back-fill history.
|
|
35
|
-
|
|
110
|
+
//
|
|
111
|
+
// `received` runs BEFORE auth so a trigger that is never admitted (bad token,
|
|
112
|
+
// disabled endpoint) still leaves durable evidence that the clock fired —
|
|
113
|
+
// "the clock never arrived" and "the clock was rejected" used to look the
|
|
114
|
+
// same from outside: nothing at all.
|
|
115
|
+
app.post('/internal/schedules/:scheduleId/run', this.received, this.auth, async (req, res) => {
|
|
36
116
|
try {
|
|
37
117
|
const scheduleId = req.params.scheduleId;
|
|
38
118
|
if (!this.handlers.listSchedules().includes(scheduleId)) {
|
|
39
119
|
res.status(404).json({ status: 'error', error: `unknown schedule: ${scheduleId}` });
|
|
120
|
+
this.triggerOutcome('schedule.trigger_not_found', 404, req, res, { schedule_id: scheduleId });
|
|
40
121
|
return;
|
|
41
122
|
}
|
|
42
123
|
// Optional human label for provenance (e.g. a deploy-layer "今日早班").
|
|
@@ -44,56 +125,79 @@ class ScheduleTriggerServer {
|
|
|
44
125
|
const label = typeof req.body?.label === 'string' ? req.body.label.slice(0, 80) : undefined;
|
|
45
126
|
const resolved = this.handlers.resolve(scheduleId, 'http', new Date(), label);
|
|
46
127
|
if (!('context' in resolved)) {
|
|
128
|
+
// A resolve refusal (expired occurrence 410, too-early 425, bad cron
|
|
129
|
+
// 400) is a business rejection, so it reports `resolved.status`
|
|
130
|
+
// rather than the transport-level 503.
|
|
47
131
|
res.status(resolved.status).json({ status: 'error', error: resolved.error });
|
|
132
|
+
this.triggerOutcome('schedule.trigger_rejected', resolved.status, req, res, {
|
|
133
|
+
schedule_id: scheduleId,
|
|
134
|
+
disposition: 'rejected',
|
|
135
|
+
reason: resolved.error,
|
|
136
|
+
});
|
|
48
137
|
return;
|
|
49
138
|
}
|
|
50
|
-
const
|
|
139
|
+
const context = resolved.context;
|
|
140
|
+
const result = await this.handlers.run(scheduleId, context);
|
|
51
141
|
// Business disposition, not HTTP luck: an accepted or already-running
|
|
52
142
|
// occurrence is NOT a completed one. Returning 200/"completed" for a run
|
|
53
143
|
// that is still executing makes an external clock stop retrying and
|
|
54
144
|
// silently lose the slot, so 'running' is reported as 202 here.
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
schedule: result,
|
|
67
|
-
note: 'queued',
|
|
68
|
-
});
|
|
69
|
-
return;
|
|
70
|
-
case 'already_running':
|
|
71
|
-
res.status(202).json({
|
|
72
|
-
status: 'running',
|
|
73
|
-
schedule: result,
|
|
74
|
-
note: 'already_running',
|
|
75
|
-
});
|
|
76
|
-
return;
|
|
77
|
-
default:
|
|
78
|
-
res.status(503).json({
|
|
79
|
-
status: 'rejected',
|
|
80
|
-
schedule: result,
|
|
81
|
-
note: 'rejected',
|
|
82
|
-
});
|
|
83
|
-
return;
|
|
84
|
-
}
|
|
145
|
+
// Status code, body and log event all come from ONE table.
|
|
146
|
+
const spec = TRIGGER_OUTCOMES[result.disposition] ?? TRIGGER_OUTCOMES.rejected;
|
|
147
|
+
res.status(spec.httpStatus).json({ status: spec.status, schedule: result, note: spec.note });
|
|
148
|
+
this.triggerOutcome(spec.event, spec.httpStatus, req, res, {
|
|
149
|
+
schedule_id: scheduleId,
|
|
150
|
+
slot_id: context.slotId,
|
|
151
|
+
occurrence_at: isoUtc(context.occurrenceAt),
|
|
152
|
+
disposition: result.disposition,
|
|
153
|
+
reason: result.cells?.find((cell) => cell.error)?.error ?? undefined,
|
|
154
|
+
trigger_source: context.triggerSource,
|
|
155
|
+
});
|
|
85
156
|
}
|
|
86
157
|
catch (error) {
|
|
87
158
|
logger_1.logger.error('Schedule trigger failed', { error: error instanceof Error ? error.message : String(error) });
|
|
88
159
|
// The slot ledger resumes on the next trigger; a 500 tells the clock to
|
|
89
160
|
// retry safely (idempotent — the same occurrence/ slot is reused).
|
|
90
161
|
res.status(500).json({ status: 'error', error: 'schedule run failed; the occurrence will resume on the next trigger' });
|
|
162
|
+
this.triggerOutcome('schedule.trigger_error', 500, req, res, {
|
|
163
|
+
schedule_id: req.params.scheduleId,
|
|
164
|
+
reason: error instanceof Error ? error.message : String(error),
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
});
|
|
168
|
+
app.post('/internal/targets/:targetId/refetch', this.refetchAuth, async (req, res) => {
|
|
169
|
+
const requestId = req.body?.requestId;
|
|
170
|
+
if (typeof requestId !== 'string' || !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(requestId)) {
|
|
171
|
+
res.status(400).json({ status: 'error', error: 'requestId must be a UUID' });
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
174
|
+
// Opaque caller correlation (review chain / review id). Optional; bounded
|
|
175
|
+
// length, never interpreted here. Recorded with the manual Slot so a
|
|
176
|
+
// recovered worker still correlates the outcome with the requester.
|
|
177
|
+
const correlationId = req.body?.correlationId;
|
|
178
|
+
if (correlationId !== undefined && (typeof correlationId !== 'string' || correlationId.length > 200)) {
|
|
179
|
+
res.status(400).json({ status: 'error', error: 'correlationId must be a string of at most 200 chars' });
|
|
180
|
+
return;
|
|
181
|
+
}
|
|
182
|
+
if (!this.handlers.refetch) {
|
|
183
|
+
res.status(503).json({ status: 'error', error: 'refetch is unavailable' });
|
|
184
|
+
return;
|
|
185
|
+
}
|
|
186
|
+
try {
|
|
187
|
+
const result = await this.handlers.refetch(req.params.targetId, requestId, correlationId ?? undefined);
|
|
188
|
+
res.status(202).json({ status: 'accepted', ...result });
|
|
189
|
+
}
|
|
190
|
+
catch (error) {
|
|
191
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
192
|
+
const status = message === 'unknown target' ? 404 : message === 'ambiguous target' ? 409 : 500;
|
|
193
|
+
logger_1.logger.warn('Manual refetch rejected', { targetId: req.params.targetId, requestId, status, error: message });
|
|
194
|
+
res.status(status).json({ status: 'error', error: status === 500 ? 'refetch admission failed' : message });
|
|
91
195
|
}
|
|
92
196
|
});
|
|
93
197
|
// Convergence endpoint: after a machine stop/start, an operator or an
|
|
94
198
|
// external watcher can ask the process to flush due deliveries/notifications
|
|
95
199
|
// without running candidate selection. Deployment-agnostic (no platform refs).
|
|
96
|
-
app.post('/internal/outbox/drain', this.auth, async (
|
|
200
|
+
app.post('/internal/outbox/drain', this.auth, async (req, res) => {
|
|
97
201
|
try {
|
|
98
202
|
if (!this.handlers.drainOutbox) {
|
|
99
203
|
res.status(503).json({ status: 'error', error: 'outbox worker not available in this runtime' });
|
|
@@ -103,7 +207,11 @@ class ScheduleTriggerServer {
|
|
|
103
207
|
res.json({ status: 'ok', result });
|
|
104
208
|
}
|
|
105
209
|
catch (error) {
|
|
106
|
-
|
|
210
|
+
// Same correlation as the trigger lines, at zero extra cost.
|
|
211
|
+
logger_1.logger.error('Outbox drain failed', {
|
|
212
|
+
...this.attemptMeta(req, res),
|
|
213
|
+
error: error instanceof Error ? error.message : String(error),
|
|
214
|
+
});
|
|
107
215
|
res.status(500).json({ status: 'error', error: 'outbox drain failed; rows remain durable and retry' });
|
|
108
216
|
}
|
|
109
217
|
});
|
|
@@ -111,18 +219,76 @@ class ScheduleTriggerServer {
|
|
|
111
219
|
logger_1.logger.info('Schedule trigger server listening', { host, port, auth: this.token ? 'bearer' : 'DISABLED (no token)' });
|
|
112
220
|
});
|
|
113
221
|
}
|
|
222
|
+
/**
|
|
223
|
+
* Per-request correlation, installed before every route. It records only the
|
|
224
|
+
* sanitized attempt id and the arrival instant, so each later line can report
|
|
225
|
+
* the same `attempt_id` and a handler-relative `elapsed_ms`.
|
|
226
|
+
*/
|
|
227
|
+
correlate = (req, res, next) => {
|
|
228
|
+
res.locals.triggerStartedAt = Date.now();
|
|
229
|
+
res.locals.triggerAttemptId = scheduleAttemptId(req);
|
|
230
|
+
res.locals.triggerProvider = scheduleProvider(req);
|
|
231
|
+
next();
|
|
232
|
+
};
|
|
233
|
+
/**
|
|
234
|
+
* `trigger.received` — the clock arrived. Runs before auth, so it carries only
|
|
235
|
+
* non-sensitive request metadata: never the Authorization header, the bearer
|
|
236
|
+
* value, the configured token, or any fragment of them.
|
|
237
|
+
*/
|
|
238
|
+
received = (req, res, next) => {
|
|
239
|
+
logger_1.logger.info('Schedule trigger received', { event: 'schedule.trigger_received', ...this.attemptMeta(req, res) });
|
|
240
|
+
next();
|
|
241
|
+
};
|
|
242
|
+
/**
|
|
243
|
+
* The single writer for every post-resolution `trigger.*` line. `event` and
|
|
244
|
+
* `http_status` are passed in from the caller's outcome table, so the line
|
|
245
|
+
* always describes the same status the response carried.
|
|
246
|
+
*/
|
|
247
|
+
triggerOutcome(event, httpStatus, req, res, extra) {
|
|
248
|
+
logger_1.logger.info('Schedule trigger outcome', {
|
|
249
|
+
event,
|
|
250
|
+
...this.attemptMeta(req, res),
|
|
251
|
+
http_status: httpStatus,
|
|
252
|
+
...extra,
|
|
253
|
+
});
|
|
254
|
+
}
|
|
255
|
+
/**
|
|
256
|
+
* The non-sensitive correlation fields shared by every `schedule.trigger_*`
|
|
257
|
+
* line. `attempt_id` and `provider` are OMITTED (never defaulted) when the
|
|
258
|
+
* clock did not send them, so an unidentified clock stays visible as such.
|
|
259
|
+
*/
|
|
260
|
+
attemptMeta(req, res) {
|
|
261
|
+
const attemptId = res.locals.triggerAttemptId;
|
|
262
|
+
const provider = res.locals.triggerProvider;
|
|
263
|
+
const startedAt = res.locals.triggerStartedAt ?? Date.now();
|
|
264
|
+
return {
|
|
265
|
+
...(attemptId ? { attempt_id: attemptId } : {}),
|
|
266
|
+
...(provider ? { provider } : {}),
|
|
267
|
+
path: req.path,
|
|
268
|
+
method: req.method,
|
|
269
|
+
elapsed_ms: Date.now() - startedAt,
|
|
270
|
+
};
|
|
271
|
+
}
|
|
114
272
|
auth = (req, res, next) => {
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
273
|
+
this.authenticate(this.token, req, res, next);
|
|
274
|
+
};
|
|
275
|
+
refetchAuth = (req, res, next) => {
|
|
276
|
+
this.authenticate(this.refetchToken, req, res, next);
|
|
277
|
+
};
|
|
278
|
+
authenticate(expected, req, res, next) {
|
|
279
|
+
if (!expected) {
|
|
280
|
+
// Fail closed: never allow an unauthenticated trigger in production. This
|
|
281
|
+
// is an admission failure, so it is reported with the status actually sent.
|
|
282
|
+
res.status(503).json({ status: 'error', error: 'endpoint disabled: token not configured' });
|
|
283
|
+
this.triggerOutcome('schedule.trigger_unauthorized', 503, req, res, { reason: 'endpoint disabled: no token configured' });
|
|
118
284
|
return;
|
|
119
285
|
}
|
|
120
286
|
const header = req.headers.authorization ?? '';
|
|
121
287
|
const presented = header.startsWith('Bearer ') ? header.slice(7).trim() : '';
|
|
122
|
-
const expected = this.token;
|
|
123
288
|
let ok = presented.length === expected.length;
|
|
124
289
|
if (ok) {
|
|
125
290
|
try {
|
|
291
|
+
// Values only, never logged: the timing check is deliberately opaque.
|
|
126
292
|
ok = (0, node_crypto_1.timingSafeEqual)(Buffer.from(presented), Buffer.from(expected));
|
|
127
293
|
}
|
|
128
294
|
catch {
|
|
@@ -131,10 +297,11 @@ class ScheduleTriggerServer {
|
|
|
131
297
|
}
|
|
132
298
|
if (!ok) {
|
|
133
299
|
res.status(401).json({ status: 'error', error: 'unauthorized' });
|
|
300
|
+
this.triggerOutcome('schedule.trigger_unauthorized', 401, req, res, { reason: 'invalid or missing bearer token' });
|
|
134
301
|
return;
|
|
135
302
|
}
|
|
136
303
|
next();
|
|
137
|
-
}
|
|
304
|
+
}
|
|
138
305
|
stop() {
|
|
139
306
|
this.server?.close();
|
|
140
307
|
this.server = null;
|
|
@@ -18,8 +18,8 @@ import { TargetExecutionContext, WorkBinding } from './WorkIdentity';
|
|
|
18
18
|
export declare const SLOT_LEASE_TTL_MS: number;
|
|
19
19
|
export declare const SLOT_HEARTBEAT_MS: number;
|
|
20
20
|
/**
|
|
21
|
-
* Durable execution context attached to a
|
|
22
|
-
*
|
|
21
|
+
* Durable execution context attached to a scheduled occurrence or a remote
|
|
22
|
+
* manual replacement. The local run-once CLI has no SlotContext.
|
|
23
23
|
*/
|
|
24
24
|
export interface SlotContext {
|
|
25
25
|
slotId: string;
|
|
@@ -36,6 +36,16 @@ export interface SlotContext {
|
|
|
36
36
|
*/
|
|
37
37
|
slotName: string;
|
|
38
38
|
slotDate: string;
|
|
39
|
+
/**
|
|
40
|
+
* Remote manual replacement ("重抓") request UUID. Present only on slots
|
|
41
|
+
* opened by the authenticated refetch endpoint; null for scheduled
|
|
42
|
+
* occurrences. Persisted with the slot so a sleeping worker that recovers
|
|
43
|
+
* the slot still knows it was a manual replacement (delivery correlation,
|
|
44
|
+
* outcome reporting) without any in-memory state.
|
|
45
|
+
*/
|
|
46
|
+
manualRequestId?: string;
|
|
47
|
+
/** Opaque caller correlation (review chain / review id). Null unless manual. */
|
|
48
|
+
correlationId?: string;
|
|
39
49
|
}
|
|
40
50
|
export interface SlotCellSummary {
|
|
41
51
|
targetId: string;
|
|
@@ -50,6 +60,54 @@ export interface SlotRunSummary {
|
|
|
50
60
|
alreadyCompleted: boolean;
|
|
51
61
|
cells: SlotCellSummary[];
|
|
52
62
|
}
|
|
63
|
+
/** Terminal statuses only: the rollup that ends an occurrence's life. */
|
|
64
|
+
export type ScheduleOutcomeStatus = 'success' | 'partial' | 'failed';
|
|
65
|
+
export interface ScheduleOutcomeTarget {
|
|
66
|
+
target_id: string;
|
|
67
|
+
status: CellStatus;
|
|
68
|
+
work_id: string | null;
|
|
69
|
+
error: string | null;
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Business categories an operator actually asks about, counted per cell. The
|
|
73
|
+
* point is to separate "nothing suitable existed" (no_match) and "already
|
|
74
|
+
* delivered" (duplicate) from real failures: lumping them together made a
|
|
75
|
+
* healthy run look broken, and a broken one look routine.
|
|
76
|
+
*
|
|
77
|
+
* Each cell lands in exactly ONE category, in this precedence:
|
|
78
|
+
* submitted -> no_match -> duplicate -> delivery_failed -> executor_failed.
|
|
79
|
+
* `delivery_failed` is checked before `executor_failed` because a terminally
|
|
80
|
+
* lost delivery ALSO leaves the cell in the `failed` state — counting it twice
|
|
81
|
+
* would invent a second failure that does not exist.
|
|
82
|
+
*/
|
|
83
|
+
export interface ScheduleOutcomeCells {
|
|
84
|
+
total: number;
|
|
85
|
+
submitted: number;
|
|
86
|
+
no_match: number;
|
|
87
|
+
duplicate: number;
|
|
88
|
+
/** True when there is at least one non-submitted cell and ALL of them are duplicates. */
|
|
89
|
+
all_duplicates: boolean;
|
|
90
|
+
executor_failed: number;
|
|
91
|
+
delivery_failed: number;
|
|
92
|
+
targets: ScheduleOutcomeTarget[];
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* The single structured terminal record for one schedule occurrence. The SAME
|
|
96
|
+
* object is both logged as `schedule.outcome` and persisted into the existing
|
|
97
|
+
* `execution.summary` detail, so the log line and the durable row cannot
|
|
98
|
+
* disagree — and no second event name or shadow ledger is introduced.
|
|
99
|
+
*/
|
|
100
|
+
export interface ScheduleOutcomeRecord {
|
|
101
|
+
event: 'schedule.outcome';
|
|
102
|
+
schedule_id: string;
|
|
103
|
+
slot_id: string;
|
|
104
|
+
occurrence_at: string | undefined;
|
|
105
|
+
occurrence_date: string;
|
|
106
|
+
status: ScheduleOutcomeStatus;
|
|
107
|
+
/** From the slot row's own started_at/completed_at columns, when both exist. */
|
|
108
|
+
duration_ms: number | undefined;
|
|
109
|
+
cells: ScheduleOutcomeCells;
|
|
110
|
+
}
|
|
53
111
|
export interface SlotResolveResult {
|
|
54
112
|
context?: SlotContext;
|
|
55
113
|
error?: string;
|
|
@@ -196,11 +254,31 @@ export declare class SlotCoordinator {
|
|
|
196
254
|
releaseRunLease(slotId: string, owner: string): void;
|
|
197
255
|
/** Roll cell results up into the slot status (one place computes the aggregate). */
|
|
198
256
|
finish(slot: SlotContext, schedule: ScheduleConfig, targets: TargetConfig[]): SlotRunSummary;
|
|
257
|
+
/**
|
|
258
|
+
* Project the terminal slot row into the one structured outcome record. Every
|
|
259
|
+
* field comes from durable state — the slot row, its cells, and the delivery
|
|
260
|
+
* ledger. Nothing is inferred from error-message text.
|
|
261
|
+
*/
|
|
262
|
+
private scheduleOutcome;
|
|
263
|
+
/**
|
|
264
|
+
* One cell -> one business category. `delivery_failed` is derived from the
|
|
265
|
+
* durable delivery ledger through the SAME port the FSM already uses, never
|
|
266
|
+
* from an error string, and it is decided BEFORE `executor_failed` because a
|
|
267
|
+
* terminally lost delivery also leaves the cell `failed`: one cause, one count.
|
|
268
|
+
*/
|
|
269
|
+
private classifyCell;
|
|
199
270
|
/**
|
|
200
271
|
* Persist a one-row incident/execution summary into delivery_events
|
|
201
272
|
* (event='execution.summary') at the terminal rollup only. Event-row storage
|
|
202
273
|
* reuses the audit table + `runs show` read path instead of inventing a
|
|
203
274
|
* summary table or overloading execution_log's illustration/novel typing.
|
|
275
|
+
*
|
|
276
|
+
* The `schedule.outcome` line is emitted HERE, immediately after that durable
|
|
277
|
+
* write and behind the SAME `hasExecutionSummary` dedupe, because the summary
|
|
278
|
+
* row is the occurrence-level terminal identity that already exists. Recovery
|
|
279
|
+
* that re-rolls the same terminal slot therefore logs nothing a second time,
|
|
280
|
+
* with no shadow ledger and no second event name. The identical object goes to
|
|
281
|
+
* both sinks, so the line and the row cannot disagree.
|
|
204
282
|
*/
|
|
205
283
|
private persistExecutionSummary;
|
|
206
284
|
completedSummary(slotId: string, schedule: ScheduleConfig): SlotRunSummary;
|
|
@@ -25,6 +25,31 @@ function deliveryTargetOf(target) {
|
|
|
25
25
|
const name = target.delivery?.target;
|
|
26
26
|
return typeof name === 'string' && name.trim() ? name.trim() : null;
|
|
27
27
|
}
|
|
28
|
+
/**
|
|
29
|
+
* SQLite `CURRENT_TIMESTAMP` is UTC "YYYY-MM-DD HH:MM:SS". It carries no zone
|
|
30
|
+
* marker, and `Date.parse` reads that shape as LOCAL time — so the zone is
|
|
31
|
+
* added explicitly instead of being trusted to the engine.
|
|
32
|
+
*/
|
|
33
|
+
function sqliteUtcMs(value) {
|
|
34
|
+
if (!value)
|
|
35
|
+
return undefined;
|
|
36
|
+
const normalized = /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/.test(value)
|
|
37
|
+
? `${value.replace(' ', 'T')}Z`
|
|
38
|
+
: value;
|
|
39
|
+
const ms = Date.parse(normalized);
|
|
40
|
+
return Number.isNaN(ms) ? undefined : ms;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Epoch ms -> ISO 8601 UTC, or undefined when it is not a usable instant.
|
|
44
|
+
* `toISOString` throws on an out-of-range Date, and a throw here would abort the
|
|
45
|
+
* terminal rollup — a logging field must never be able to break dispatch.
|
|
46
|
+
*/
|
|
47
|
+
function isoUtcOrUndefined(epochMs) {
|
|
48
|
+
if (typeof epochMs !== 'number' || !Number.isFinite(epochMs))
|
|
49
|
+
return undefined;
|
|
50
|
+
const date = new Date(epochMs);
|
|
51
|
+
return Number.isNaN(date.getTime()) ? undefined : date.toISOString();
|
|
52
|
+
}
|
|
28
53
|
/**
|
|
29
54
|
* Owns the Schedule Slot ledger for one scheduler run. A Slot is one durable
|
|
30
55
|
* execution occurrence of a Schedule (NOT a morning/evening row). It ensures
|
|
@@ -102,6 +127,8 @@ class SlotCoordinator {
|
|
|
102
127
|
triggerSource: slot.triggerSource,
|
|
103
128
|
slotDate: slot.slotDate,
|
|
104
129
|
slotName: slot.slotName,
|
|
130
|
+
manualRequestId: slot.manualRequestId ?? null,
|
|
131
|
+
correlationId: slot.correlationId ?? null,
|
|
105
132
|
});
|
|
106
133
|
if (created) {
|
|
107
134
|
// Freeze membership: materialize one cell per target id from the snapshot.
|
|
@@ -347,25 +374,139 @@ class SlotCoordinator {
|
|
|
347
374
|
}
|
|
348
375
|
const status = this.database.slots.deriveSlotStatus(slot.slotId);
|
|
349
376
|
this.database.slots.markSlotStatus(slot.slotId, status);
|
|
350
|
-
this.persistExecutionSummary(slot.slotId, status);
|
|
351
377
|
const cells = this.database.slots.getCells(slot.slotId).map((c) => ({
|
|
352
378
|
targetId: c.targetId,
|
|
353
379
|
status: c.status,
|
|
354
380
|
workId: c.workId,
|
|
355
381
|
error: c.lastError,
|
|
356
382
|
}));
|
|
383
|
+
// Rolled up AFTER the slot row carries its terminal status/completed_at, so
|
|
384
|
+
// the outcome reports the durable timestamps rather than a fresh clock read.
|
|
385
|
+
this.persistExecutionSummary(slot.slotId, status, this.scheduleOutcome(slot, status, targets));
|
|
357
386
|
const icon = (s) => s === 'submitted' ? '✅' : s === 'no_candidate' ? '⚠️ no_candidate' : '❌ failed';
|
|
358
387
|
logger_1.logger.info(`Slot ${slot.slotId} ${status}\n` +
|
|
359
388
|
cells.map((c) => ` ${c.targetId.padEnd(28)} ${icon(c.status)} ${c.workId ? '#' + c.workId : ''} ${c.error ? '(' + c.error + ')' : ''}`).join('\n'), { slot: slot.slotId, status });
|
|
360
389
|
return { scheduleId: schedule.id, slotId: slot.slotId, status, alreadyCompleted: false, cells };
|
|
361
390
|
}
|
|
391
|
+
/**
|
|
392
|
+
* Project the terminal slot row into the one structured outcome record. Every
|
|
393
|
+
* field comes from durable state — the slot row, its cells, and the delivery
|
|
394
|
+
* ledger. Nothing is inferred from error-message text.
|
|
395
|
+
*/
|
|
396
|
+
scheduleOutcome(slot, status, targets) {
|
|
397
|
+
const slotRec = this.database.slots.getSlot(slot.slotId);
|
|
398
|
+
const cellRows = this.database.slots.getCells(slot.slotId);
|
|
399
|
+
// Only targets the caller still knows about can name a delivery channel; a
|
|
400
|
+
// cell with no known channel has no delivery fact to consult.
|
|
401
|
+
const deliveryTargetByTargetId = new Map();
|
|
402
|
+
for (const target of targets) {
|
|
403
|
+
const deliveryTarget = deliveryTargetOf(target);
|
|
404
|
+
if (target.id && deliveryTarget)
|
|
405
|
+
deliveryTargetByTargetId.set(target.id, deliveryTarget);
|
|
406
|
+
}
|
|
407
|
+
let submitted = 0;
|
|
408
|
+
let no_match = 0;
|
|
409
|
+
let duplicate = 0;
|
|
410
|
+
let executor_failed = 0;
|
|
411
|
+
let delivery_failed = 0;
|
|
412
|
+
const targetsDetail = cellRows.map((cell) => {
|
|
413
|
+
const classified = this.classifyCell(cell.status, slot.slotId, cell.targetId, deliveryTargetByTargetId);
|
|
414
|
+
if (classified === 'submitted')
|
|
415
|
+
submitted += 1;
|
|
416
|
+
else if (classified === 'no_match')
|
|
417
|
+
no_match += 1;
|
|
418
|
+
else if (classified === 'duplicate')
|
|
419
|
+
duplicate += 1;
|
|
420
|
+
else if (classified === 'delivery_failed')
|
|
421
|
+
delivery_failed += 1;
|
|
422
|
+
else if (classified === 'executor_failed')
|
|
423
|
+
executor_failed += 1;
|
|
424
|
+
return {
|
|
425
|
+
target_id: cell.targetId,
|
|
426
|
+
status: cell.status,
|
|
427
|
+
work_id: cell.workId,
|
|
428
|
+
error: cell.lastError,
|
|
429
|
+
};
|
|
430
|
+
});
|
|
431
|
+
// A fully-submitted slot has no non-submitted cell at all, and reporting
|
|
432
|
+
// `all_duplicates` there would be a lie — so the count must be non-zero.
|
|
433
|
+
const nonSubmitted = cellRows.length - submitted;
|
|
434
|
+
const all_duplicates = nonSubmitted > 0 && duplicate === nonSubmitted;
|
|
435
|
+
const startedMs = sqliteUtcMs(slotRec?.startedAt);
|
|
436
|
+
const completedMs = sqliteUtcMs(slotRec?.completedAt);
|
|
437
|
+
const duration_ms = startedMs !== undefined && completedMs !== undefined
|
|
438
|
+
? Math.max(0, completedMs - startedMs)
|
|
439
|
+
: undefined;
|
|
440
|
+
const occurrenceAt = slotRec?.occurrenceAt ?? slot.occurrenceAt;
|
|
441
|
+
return {
|
|
442
|
+
event: 'schedule.outcome',
|
|
443
|
+
schedule_id: slotRec?.scheduleId ?? slot.scheduleId,
|
|
444
|
+
slot_id: slot.slotId,
|
|
445
|
+
occurrence_at: isoUtcOrUndefined(occurrenceAt),
|
|
446
|
+
occurrence_date: slotRec?.occurrenceDate ?? slot.occurrenceDate,
|
|
447
|
+
status: status,
|
|
448
|
+
duration_ms,
|
|
449
|
+
cells: {
|
|
450
|
+
total: cellRows.length,
|
|
451
|
+
submitted,
|
|
452
|
+
no_match,
|
|
453
|
+
duplicate,
|
|
454
|
+
all_duplicates,
|
|
455
|
+
executor_failed,
|
|
456
|
+
delivery_failed,
|
|
457
|
+
targets: targetsDetail,
|
|
458
|
+
},
|
|
459
|
+
};
|
|
460
|
+
}
|
|
461
|
+
/**
|
|
462
|
+
* One cell -> one business category. `delivery_failed` is derived from the
|
|
463
|
+
* durable delivery ledger through the SAME port the FSM already uses, never
|
|
464
|
+
* from an error string, and it is decided BEFORE `executor_failed` because a
|
|
465
|
+
* terminally lost delivery also leaves the cell `failed`: one cause, one count.
|
|
466
|
+
*/
|
|
467
|
+
classifyCell(cellStatus, slotId, targetId, deliveryTargetByTargetId) {
|
|
468
|
+
if (cellStatus === 'submitted')
|
|
469
|
+
return 'submitted';
|
|
470
|
+
if (cellStatus === 'no_candidate')
|
|
471
|
+
return 'no_match';
|
|
472
|
+
if (cellStatus === 'duplicate')
|
|
473
|
+
return 'duplicate';
|
|
474
|
+
const deliveryTarget = deliveryTargetByTargetId.get(targetId);
|
|
475
|
+
if (deliveryTarget && this.delivery) {
|
|
476
|
+
try {
|
|
477
|
+
const state = this.delivery.stateFor({ deliveryTarget, slotId, targetId });
|
|
478
|
+
if (state.kind === 'lost')
|
|
479
|
+
return 'delivery_failed';
|
|
480
|
+
}
|
|
481
|
+
catch (error) {
|
|
482
|
+
// Observability must never break the rollup.
|
|
483
|
+
logger_1.logger.debug('Delivery state unavailable for outcome rollup', {
|
|
484
|
+
slot: slotId,
|
|
485
|
+
target: targetId,
|
|
486
|
+
error: error.message,
|
|
487
|
+
});
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
if (cellStatus === 'failed')
|
|
491
|
+
return 'executor_failed';
|
|
492
|
+
// Non-terminal (delivery_pending / artifact_ready): such a slot is not
|
|
493
|
+
// terminal either, so no outcome record is written for it at all.
|
|
494
|
+
return 'other';
|
|
495
|
+
}
|
|
362
496
|
/**
|
|
363
497
|
* Persist a one-row incident/execution summary into delivery_events
|
|
364
498
|
* (event='execution.summary') at the terminal rollup only. Event-row storage
|
|
365
499
|
* reuses the audit table + `runs show` read path instead of inventing a
|
|
366
500
|
* summary table or overloading execution_log's illustration/novel typing.
|
|
501
|
+
*
|
|
502
|
+
* The `schedule.outcome` line is emitted HERE, immediately after that durable
|
|
503
|
+
* write and behind the SAME `hasExecutionSummary` dedupe, because the summary
|
|
504
|
+
* row is the occurrence-level terminal identity that already exists. Recovery
|
|
505
|
+
* that re-rolls the same terminal slot therefore logs nothing a second time,
|
|
506
|
+
* with no shadow ledger and no second event name. The identical object goes to
|
|
507
|
+
* both sinks, so the line and the row cannot disagree.
|
|
367
508
|
*/
|
|
368
|
-
persistExecutionSummary(slotId, status) {
|
|
509
|
+
persistExecutionSummary(slotId, status, outcome) {
|
|
369
510
|
if (status !== 'success' && status !== 'partial' && status !== 'failed')
|
|
370
511
|
return;
|
|
371
512
|
try {
|
|
@@ -377,8 +518,9 @@ class SlotCoordinator {
|
|
|
377
518
|
slotId,
|
|
378
519
|
event: 'execution.summary',
|
|
379
520
|
countsAsAttempt: 0,
|
|
380
|
-
detail: { summary },
|
|
521
|
+
detail: { summary, outcome },
|
|
381
522
|
});
|
|
523
|
+
logger_1.logger.info('Schedule occurrence reached a terminal outcome', outcome);
|
|
382
524
|
}
|
|
383
525
|
catch (error) {
|
|
384
526
|
logger_1.logger.debug('Failed to persist execution summary', { slot: slotId, error: error.message });
|
|
@@ -99,6 +99,15 @@ class DatabaseMigration {
|
|
|
99
99
|
lease_owner TEXT,
|
|
100
100
|
lease_until INTEGER,
|
|
101
101
|
heartbeat_at INTEGER,
|
|
102
|
+
-- Request UUID of a remote manual replacement ("重抓"). NULL for a
|
|
103
|
+
-- scheduled occurrence. Persisted in the SAME transaction that
|
|
104
|
+
-- opens the manual slot, so the acceptance ACK and the caller's
|
|
105
|
+
-- retry converge on one row instead of racing a second slot.
|
|
106
|
+
manual_request_id TEXT,
|
|
107
|
+
-- Opaque caller correlation (review chain / review id). Recorded so
|
|
108
|
+
-- the terminal outcome can be reported back to the requester
|
|
109
|
+
-- without this service learning anything about the review.
|
|
110
|
+
correlation_id TEXT,
|
|
102
111
|
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
103
112
|
started_at DATETIME,
|
|
104
113
|
completed_at DATETIME,
|
|
@@ -227,6 +236,8 @@ class DatabaseMigration {
|
|
|
227
236
|
lease_owner: 'ALTER TABLE schedule_slots ADD COLUMN lease_owner TEXT',
|
|
228
237
|
lease_until: 'ALTER TABLE schedule_slots ADD COLUMN lease_until INTEGER',
|
|
229
238
|
heartbeat_at: 'ALTER TABLE schedule_slots ADD COLUMN heartbeat_at INTEGER',
|
|
239
|
+
manual_request_id: 'ALTER TABLE schedule_slots ADD COLUMN manual_request_id TEXT',
|
|
240
|
+
correlation_id: 'ALTER TABLE schedule_slots ADD COLUMN correlation_id TEXT',
|
|
230
241
|
};
|
|
231
242
|
const columnAlters = [];
|
|
232
243
|
for (const [col, sql] of Object.entries(slotColumnMigrations)) {
|