pixivflow 2.20.5 → 2.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +179 -56
- package/README.md +110 -57
- package/dist/commands/SchedulerCommand.js +63 -0
- package/dist/commands/SchedulerIdleLifecycle.d.ts +8 -0
- package/dist/commands/SchedulerIdleLifecycle.js +2 -1
- package/dist/commands/scheduler-runtime.js +17 -0
- package/dist/config/types.d.ts +26 -0
- package/dist/config/validation.js +21 -0
- package/dist/delivery/types.d.ts +15 -0
- package/dist/notification/NotificationPolicy.d.ts +2 -0
- package/dist/notification/NotificationPolicy.js +18 -1
- package/dist/package.json +1 -1
- package/dist/scheduler/MultiScheduleManager.d.ts +12 -1
- package/dist/scheduler/MultiScheduleManager.js +46 -50
- package/dist/scheduler/RecoveryPolicy.d.ts +45 -0
- package/dist/scheduler/RecoveryPolicy.js +63 -0
- package/dist/scheduler/ResourceAdmission.d.ts +68 -0
- package/dist/scheduler/ResourceAdmission.js +83 -0
- package/dist/scheduler/ScheduleTriggerServer.d.ts +16 -0
- package/dist/scheduler/ScheduleTriggerServer.js +55 -0
- package/dist/scheduler/Scheduler.d.ts +19 -2
- package/dist/scheduler/Scheduler.js +12 -3
- package/dist/scheduler/SlotCoordinator.d.ts +20 -0
- package/dist/scheduler/SlotCoordinator.js +44 -2
- package/dist/scheduler/TargetOutcome.d.ts +31 -0
- package/dist/scheduler/TargetOutcome.js +128 -0
- package/dist/storage/DatabaseMigration.js +17 -0
- package/dist/storage/repositories/SlotRepository.d.ts +29 -0
- package/dist/storage/repositories/SlotRepository.js +28 -2
- package/dist/version.js +1 -1
- package/dist/webui/package.json +1 -1
- package/package.json +2 -2
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ResourceAdmission = void 0;
|
|
4
|
+
exports.pixivAccountResourceKey = pixivAccountResourceKey;
|
|
5
|
+
/**
|
|
6
|
+
* FIFO per-resource admission with config-driven capacity.
|
|
7
|
+
*
|
|
8
|
+
* `capacityOf(resourceKey)` returns the configured capacity for that key
|
|
9
|
+
* (default 1). `maxWaiting` bounds the TOTAL number of parked work items
|
|
10
|
+
* across all resources, mirroring the legacy `schedulerRuntime.queueLimit`
|
|
11
|
+
* guard against an unbounded backlog.
|
|
12
|
+
*/
|
|
13
|
+
class ResourceAdmission {
|
|
14
|
+
capacityOf;
|
|
15
|
+
maxWaiting;
|
|
16
|
+
active = new Map();
|
|
17
|
+
waiting = [];
|
|
18
|
+
constructor(capacityOf, maxWaiting = 8) {
|
|
19
|
+
this.capacityOf = capacityOf;
|
|
20
|
+
this.maxWaiting = maxWaiting;
|
|
21
|
+
}
|
|
22
|
+
setMaxWaiting(maxWaiting) {
|
|
23
|
+
this.maxWaiting = Math.max(0, maxWaiting);
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Acquire a lease on `resourceKey`. Resolves immediately when capacity is
|
|
27
|
+
* available, parks the caller in a FIFO per-key queue otherwise, and resolves
|
|
28
|
+
* `null` (never rejects) when the bounded wait queue is full.
|
|
29
|
+
*/
|
|
30
|
+
acquire(resourceKey) {
|
|
31
|
+
if (this.activeCount(resourceKey) < this.capacityOf(resourceKey)) {
|
|
32
|
+
this.active.set(resourceKey, this.activeCount(resourceKey) + 1);
|
|
33
|
+
return Promise.resolve(this.createLease(resourceKey));
|
|
34
|
+
}
|
|
35
|
+
if (this.waiting.length >= this.maxWaiting) {
|
|
36
|
+
return Promise.resolve(null);
|
|
37
|
+
}
|
|
38
|
+
return new Promise((resolve) => {
|
|
39
|
+
this.waiting.push({ resourceKey, resolve });
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
createLease(resourceKey) {
|
|
43
|
+
let released = false;
|
|
44
|
+
return {
|
|
45
|
+
release: () => {
|
|
46
|
+
if (released)
|
|
47
|
+
return;
|
|
48
|
+
released = true;
|
|
49
|
+
this.releaseNext(resourceKey);
|
|
50
|
+
},
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
releaseNext(resourceKey) {
|
|
54
|
+
// Hand the freed slot to the OLDEST waiter for this resource key (FIFO),
|
|
55
|
+
// skipping waiters of other resources so they never block this one.
|
|
56
|
+
const index = this.waiting.findIndex((entry) => entry.resourceKey === resourceKey);
|
|
57
|
+
if (index === -1) {
|
|
58
|
+
const remaining = Math.max(0, this.activeCount(resourceKey) - 1);
|
|
59
|
+
this.active.set(resourceKey, remaining);
|
|
60
|
+
return;
|
|
61
|
+
}
|
|
62
|
+
const [entry] = this.waiting.splice(index, 1);
|
|
63
|
+
entry.resolve(this.createLease(resourceKey));
|
|
64
|
+
}
|
|
65
|
+
/** Active leases for one resource key. */
|
|
66
|
+
activeCount(resourceKey) {
|
|
67
|
+
return this.active.get(resourceKey) ?? 0;
|
|
68
|
+
}
|
|
69
|
+
/** Work items parked waiting for one resource key. */
|
|
70
|
+
waitingCount(resourceKey) {
|
|
71
|
+
return this.waiting.filter((entry) => entry.resourceKey === resourceKey).length;
|
|
72
|
+
}
|
|
73
|
+
/** Work items parked waiting for ANY resource. Unfinished work, not failure. */
|
|
74
|
+
waitingTotal() {
|
|
75
|
+
return this.waiting.length;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
exports.ResourceAdmission = ResourceAdmission;
|
|
79
|
+
/** The canonical resource identity for the process's Pixiv credential profile. */
|
|
80
|
+
function pixivAccountResourceKey(accountId) {
|
|
81
|
+
return `pixiv-account:${(accountId ?? 'default').trim() || 'default'}`;
|
|
82
|
+
}
|
|
83
|
+
//# sourceMappingURL=ResourceAdmission.js.map
|
|
@@ -111,6 +111,22 @@ export interface TriggerHandlers {
|
|
|
111
111
|
state: string;
|
|
112
112
|
slotStatus: string;
|
|
113
113
|
} | null;
|
|
114
|
+
/**
|
|
115
|
+
* Admit one manual RECOVERY of a failed target (§manual-recovery). `retryMode`
|
|
116
|
+
* selects a SERVER-DEFINED acquisition policy preset ('normal' | 'relaxed');
|
|
117
|
+
* callers never supply raw acquisition parameters. The run is
|
|
118
|
+
* occurrence-scoped: it never changes the global config or future schedules.
|
|
119
|
+
*/
|
|
120
|
+
recover?(targetId: string, requestId: string, retryMode: 'normal' | 'relaxed', correlationId?: string): Promise<{
|
|
121
|
+
slotId: string;
|
|
122
|
+
disposition: string;
|
|
123
|
+
}>;
|
|
124
|
+
recoverStatus?(targetId: string, requestId: string): {
|
|
125
|
+
requestId: string;
|
|
126
|
+
slotId: string;
|
|
127
|
+
state: string;
|
|
128
|
+
slotStatus: string;
|
|
129
|
+
} | null;
|
|
114
130
|
}
|
|
115
131
|
export declare class ScheduleTriggerServer {
|
|
116
132
|
private readonly token;
|
|
@@ -207,6 +207,61 @@ class ScheduleTriggerServer {
|
|
|
207
207
|
}
|
|
208
208
|
res.json(status);
|
|
209
209
|
});
|
|
210
|
+
// Manual RECOVERY of a failed schedule target (§manual-recovery). Same
|
|
211
|
+
// authenticated manual-work family as refetch (PIXIVFLOW_REFETCH_TOKEN) but
|
|
212
|
+
// a DIFFERENT business intent: it re-runs the failed target(s) under a
|
|
213
|
+
// server-defined acquisition policy preset and reports through the
|
|
214
|
+
// schedule-outcome channel, so the daily summary is never rewritten.
|
|
215
|
+
// Admitting is all this endpoint does; if the resource is busy the run is
|
|
216
|
+
// queued (202 + disposition 'queued'), never failed.
|
|
217
|
+
app.post('/internal/targets/:targetId/recover', this.refetchAuth, async (req, res) => {
|
|
218
|
+
const requestId = req.body?.requestId;
|
|
219
|
+
if (typeof requestId !== 'string' || !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(requestId)) {
|
|
220
|
+
res.status(400).json({ status: 'error', error: 'requestId must be a UUID' });
|
|
221
|
+
return;
|
|
222
|
+
}
|
|
223
|
+
// Only a NAMED preset is accepted — never raw acquisition parameters.
|
|
224
|
+
const rawMode = req.body?.retryMode;
|
|
225
|
+
const retryMode = rawMode === undefined || rawMode === null || rawMode === '' ? 'normal' : rawMode;
|
|
226
|
+
if (retryMode !== 'normal' && retryMode !== 'relaxed') {
|
|
227
|
+
res.status(400).json({ status: 'error', error: 'retryMode must be "normal" or "relaxed"' });
|
|
228
|
+
return;
|
|
229
|
+
}
|
|
230
|
+
const correlationId = req.body?.correlationId;
|
|
231
|
+
if (correlationId !== undefined && (typeof correlationId !== 'string' || correlationId.length > 200)) {
|
|
232
|
+
res.status(400).json({ status: 'error', error: 'correlationId must be a string of at most 200 chars' });
|
|
233
|
+
return;
|
|
234
|
+
}
|
|
235
|
+
if (!this.handlers.recover) {
|
|
236
|
+
res.status(503).json({ status: 'error', error: 'manual recovery is unavailable' });
|
|
237
|
+
return;
|
|
238
|
+
}
|
|
239
|
+
try {
|
|
240
|
+
const result = await this.handlers.recover(req.params.targetId, requestId, retryMode, correlationId ?? undefined);
|
|
241
|
+
res.status(202).json({ status: 'accepted', retryMode, ...result });
|
|
242
|
+
}
|
|
243
|
+
catch (error) {
|
|
244
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
245
|
+
const status = message === 'unknown target' ? 404 : message === 'ambiguous target' ? 409 : 500;
|
|
246
|
+
logger_1.logger.warn('Manual recovery rejected', {
|
|
247
|
+
targetId: req.params.targetId, requestId, retryMode, status, error: message,
|
|
248
|
+
});
|
|
249
|
+
res.status(status).json({ status: 'error', error: status === 500 ? 'recovery admission failed' : message });
|
|
250
|
+
}
|
|
251
|
+
});
|
|
252
|
+
app.get('/internal/targets/:targetId/recover/:requestId', this.refetchAuth, (req, res) => {
|
|
253
|
+
const { targetId, requestId } = req.params;
|
|
254
|
+
if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(requestId)) {
|
|
255
|
+
res.status(400).json({ status: 'error', error: 'requestId must be a UUID' });
|
|
256
|
+
return;
|
|
257
|
+
}
|
|
258
|
+
const status = this.handlers.recoverStatus?.(targetId, requestId);
|
|
259
|
+
if (!status) {
|
|
260
|
+
res.status(404).json({ status: 'error', error: 'manual recovery not found' });
|
|
261
|
+
return;
|
|
262
|
+
}
|
|
263
|
+
res.json(status);
|
|
264
|
+
});
|
|
210
265
|
// Convergence endpoint: after a machine stop/start, an operator or an
|
|
211
266
|
// external watcher can ask the process to flush due deliveries/notifications
|
|
212
267
|
// without running candidate selection. Deployment-agnostic (no platform refs).
|
|
@@ -56,7 +56,12 @@ export interface JobAbandoned {
|
|
|
56
56
|
}
|
|
57
57
|
/** Admission is acquired before timeout/accounting starts. */
|
|
58
58
|
export interface JobAdmissionController {
|
|
59
|
-
|
|
59
|
+
/**
|
|
60
|
+
* Acquire a lease on the work item's constrained RESOURCE. The key is a
|
|
61
|
+
* resource identity (`pixiv-account:<id>`), never a bot/schedule/target
|
|
62
|
+
* name. Resolves `null` only when the bounded wait queue is full.
|
|
63
|
+
*/
|
|
64
|
+
acquire(resourceKey: string): Promise<JobLease | null>;
|
|
60
65
|
}
|
|
61
66
|
export declare class Scheduler {
|
|
62
67
|
private readonly config;
|
|
@@ -66,6 +71,12 @@ export declare class Scheduler {
|
|
|
66
71
|
private readonly admission?;
|
|
67
72
|
private readonly onFailure?;
|
|
68
73
|
private readonly onAbandoned?;
|
|
74
|
+
/**
|
|
75
|
+
* The constrained resource this schedule's work consumes (§resource-
|
|
76
|
+
* governance). All schedules whose targets share the same Pixiv account
|
|
77
|
+
* share the same key, so their runs queue under ONE capacity.
|
|
78
|
+
*/
|
|
79
|
+
private readonly resourceKey?;
|
|
69
80
|
private task;
|
|
70
81
|
private job;
|
|
71
82
|
private running;
|
|
@@ -76,7 +87,13 @@ export declare class Scheduler {
|
|
|
76
87
|
private drainHandle;
|
|
77
88
|
private stopped;
|
|
78
89
|
private pending;
|
|
79
|
-
constructor(config: SchedulerConfig, database?: Database | undefined, telemetry?: JobTelemetry | undefined, scheduleId?: string, admission?: JobAdmissionController | undefined, onFailure?: ((failure: JobFailure) => Promise<void> | void) | undefined, onAbandoned?: ((abandoned: JobAbandoned) => Promise<void> | void) | undefined
|
|
90
|
+
constructor(config: SchedulerConfig, database?: Database | undefined, telemetry?: JobTelemetry | undefined, scheduleId?: string, admission?: JobAdmissionController | undefined, onFailure?: ((failure: JobFailure) => Promise<void> | void) | undefined, onAbandoned?: ((abandoned: JobAbandoned) => Promise<void> | void) | undefined,
|
|
91
|
+
/**
|
|
92
|
+
* The constrained resource this schedule's work consumes (§resource-
|
|
93
|
+
* governance). All schedules whose targets share the same Pixiv account
|
|
94
|
+
* share the same key, so their runs queue under ONE capacity.
|
|
95
|
+
*/
|
|
96
|
+
resourceKey?: string | undefined);
|
|
80
97
|
start(job: (options?: ScheduleRunOptions) => Promise<void>): void;
|
|
81
98
|
/**
|
|
82
99
|
* Arm the scheduler for external/manual triggers (runNow) WITHOUT registering
|
|
@@ -33,6 +33,7 @@ class Scheduler {
|
|
|
33
33
|
admission;
|
|
34
34
|
onFailure;
|
|
35
35
|
onAbandoned;
|
|
36
|
+
resourceKey;
|
|
36
37
|
task = null;
|
|
37
38
|
job = null;
|
|
38
39
|
running = false;
|
|
@@ -43,7 +44,13 @@ class Scheduler {
|
|
|
43
44
|
drainHandle = null;
|
|
44
45
|
stopped = false;
|
|
45
46
|
pending = false;
|
|
46
|
-
constructor(config, database, telemetry, scheduleId = 'default', admission, onFailure, onAbandoned
|
|
47
|
+
constructor(config, database, telemetry, scheduleId = 'default', admission, onFailure, onAbandoned,
|
|
48
|
+
/**
|
|
49
|
+
* The constrained resource this schedule's work consumes (§resource-
|
|
50
|
+
* governance). All schedules whose targets share the same Pixiv account
|
|
51
|
+
* share the same key, so their runs queue under ONE capacity.
|
|
52
|
+
*/
|
|
53
|
+
resourceKey) {
|
|
47
54
|
this.config = config;
|
|
48
55
|
this.database = database;
|
|
49
56
|
this.telemetry = telemetry;
|
|
@@ -51,6 +58,7 @@ class Scheduler {
|
|
|
51
58
|
this.admission = admission;
|
|
52
59
|
this.onFailure = onFailure;
|
|
53
60
|
this.onAbandoned = onAbandoned;
|
|
61
|
+
this.resourceKey = resourceKey;
|
|
54
62
|
}
|
|
55
63
|
start(job) {
|
|
56
64
|
this.arm(job, true);
|
|
@@ -149,15 +157,16 @@ class Scheduler {
|
|
|
149
157
|
await new Promise((resolve) => setTimeout(resolve, this.config.failureRetryDelay));
|
|
150
158
|
}
|
|
151
159
|
this.pending = true;
|
|
152
|
-
const lease = this.admission ? await this.admission.acquire(this.scheduleId) : null;
|
|
160
|
+
const lease = this.admission ? await this.admission.acquire(this.resourceKey ?? this.scheduleId) : null;
|
|
153
161
|
this.pending = false;
|
|
154
162
|
if (this.stopped) {
|
|
155
163
|
lease?.release();
|
|
156
164
|
return;
|
|
157
165
|
}
|
|
158
166
|
if (this.admission && !lease) {
|
|
159
|
-
logger_1.logger.warn('
|
|
167
|
+
logger_1.logger.warn('Work not admitted: the resource wait queue is full; the durable ledger will retry it', {
|
|
160
168
|
scheduleId: this.scheduleId,
|
|
169
|
+
resourceKey: this.resourceKey,
|
|
161
170
|
});
|
|
162
171
|
return;
|
|
163
172
|
}
|
|
@@ -46,12 +46,26 @@ export interface SlotContext {
|
|
|
46
46
|
manualRequestId?: string;
|
|
47
47
|
/** Opaque caller correlation (review chain / review id). Null unless manual. */
|
|
48
48
|
correlationId?: string;
|
|
49
|
+
/**
|
|
50
|
+
* Remote manual RECOVERY request UUID (§manual-recovery). Present only on
|
|
51
|
+
* slots opened by the authenticated recover endpoint; null for scheduled
|
|
52
|
+
* occurrences and review refetches. A recovery slot re-runs the failed
|
|
53
|
+
* target(s) under an occurrence-scoped recovery policy and reports through
|
|
54
|
+
* the schedule-outcome channel.
|
|
55
|
+
*/
|
|
56
|
+
recoveryRequestId?: string;
|
|
57
|
+
/** Recovery policy preset ('normal' | 'relaxed'); null for non-recovery slots. */
|
|
58
|
+
recoveryMode?: 'normal' | 'relaxed';
|
|
49
59
|
}
|
|
50
60
|
export interface SlotCellSummary {
|
|
51
61
|
targetId: string;
|
|
52
62
|
status: CellStatus;
|
|
53
63
|
workId: string | null;
|
|
54
64
|
error?: string | null;
|
|
65
|
+
/** Normalized terminal failure code (§terminal-reason); null when not failed. */
|
|
66
|
+
terminalReasonCode?: string | null;
|
|
67
|
+
/** User-facing business reason message (§terminal-reason). */
|
|
68
|
+
terminalReasonMessage?: string | null;
|
|
55
69
|
}
|
|
56
70
|
export interface SlotRunSummary {
|
|
57
71
|
scheduleId: string;
|
|
@@ -233,6 +247,12 @@ export declare class SlotCoordinator {
|
|
|
233
247
|
* sole path to the submitted cell state.
|
|
234
248
|
*/
|
|
235
249
|
applyOutcome(slotId: string, targetId: string, outcome: TargetOutcome): void;
|
|
250
|
+
/**
|
|
251
|
+
* Persist the normalized terminal reason (§terminal-reason) for a cell that
|
|
252
|
+
* just reached a terminal state. Never unwinds the run and never leaks raw
|
|
253
|
+
* error internals: only the stable code + business message are stored.
|
|
254
|
+
*/
|
|
255
|
+
private persistTerminalReason;
|
|
236
256
|
/** Promote a delivery_pending cell to submitted from a confirmed ACK. */
|
|
237
257
|
markDelivered(slotId: string, targetId: string, workId: string, workType: string): void;
|
|
238
258
|
/**
|
|
@@ -4,6 +4,7 @@ exports.SlotCoordinator = exports.SLOT_HEARTBEAT_MS = exports.SLOT_LEASE_TTL_MS
|
|
|
4
4
|
exports.timezoneForSchedule = timezoneForSchedule;
|
|
5
5
|
const logger_1 = require("../logger");
|
|
6
6
|
const OccurrenceResolver_1 = require("./OccurrenceResolver");
|
|
7
|
+
const TargetOutcome_1 = require("./TargetOutcome");
|
|
7
8
|
const WorkIdentity_1 = require("./WorkIdentity");
|
|
8
9
|
/**
|
|
9
10
|
* Execution-lease TTL and heartbeat cadence.
|
|
@@ -129,6 +130,8 @@ class SlotCoordinator {
|
|
|
129
130
|
slotName: slot.slotName,
|
|
130
131
|
manualRequestId: slot.manualRequestId ?? null,
|
|
131
132
|
correlationId: slot.correlationId ?? null,
|
|
133
|
+
recoveryRequestId: slot.recoveryRequestId ?? null,
|
|
134
|
+
recoveryMode: slot.recoveryMode ?? null,
|
|
132
135
|
});
|
|
133
136
|
if (created) {
|
|
134
137
|
// Freeze membership: materialize one cell per target id from the snapshot.
|
|
@@ -267,12 +270,15 @@ class SlotCoordinator {
|
|
|
267
270
|
case 'no_candidate':
|
|
268
271
|
// Only terminal if the cell never locked a work; a locked work whose
|
|
269
272
|
// delivery is still pending must not be collapsed to no_candidate.
|
|
270
|
-
if (!cell.workId)
|
|
273
|
+
if (!cell.workId) {
|
|
271
274
|
this.safeTransition(slotId, targetId, 'no_candidate', outcome.reason);
|
|
275
|
+
this.persistTerminalReason(slotId, targetId, outcome);
|
|
276
|
+
}
|
|
272
277
|
return;
|
|
273
278
|
case 'duplicate':
|
|
274
279
|
this.database.slots.lockCellWork(slotId, targetId, outcome.workId, cell.workType ?? 'unknown');
|
|
275
280
|
this.safeTransition(slotId, targetId, 'duplicate', outcome.reason);
|
|
281
|
+
this.persistTerminalReason(slotId, targetId, outcome);
|
|
276
282
|
return;
|
|
277
283
|
case 'failed':
|
|
278
284
|
if (outcome.retryable) {
|
|
@@ -282,7 +288,27 @@ class SlotCoordinator {
|
|
|
282
288
|
return;
|
|
283
289
|
}
|
|
284
290
|
this.safeTransition(slotId, targetId, 'failed', outcome.error);
|
|
291
|
+
this.persistTerminalReason(slotId, targetId, outcome);
|
|
292
|
+
return;
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
/**
|
|
296
|
+
* Persist the normalized terminal reason (§terminal-reason) for a cell that
|
|
297
|
+
* just reached a terminal state. Never unwinds the run and never leaks raw
|
|
298
|
+
* error internals: only the stable code + business message are stored.
|
|
299
|
+
*/
|
|
300
|
+
persistTerminalReason(slotId, targetId, outcome) {
|
|
301
|
+
try {
|
|
302
|
+
const reason = (0, TargetOutcome_1.terminalReasonFor)(outcome);
|
|
303
|
+
if (!reason)
|
|
285
304
|
return;
|
|
305
|
+
this.database.slots.setCellTerminalReason(slotId, targetId, reason.code, reason.message);
|
|
306
|
+
}
|
|
307
|
+
catch (error) {
|
|
308
|
+
logger_1.logger.debug('Failed to persist terminal reason', {
|
|
309
|
+
slot: slotId, target: targetId,
|
|
310
|
+
error: error instanceof Error ? error.message : String(error),
|
|
311
|
+
});
|
|
286
312
|
}
|
|
287
313
|
}
|
|
288
314
|
/** Promote a delivery_pending cell to submitted from a confirmed ACK. */
|
|
@@ -393,7 +419,19 @@ class SlotCoordinator {
|
|
|
393
419
|
continue;
|
|
394
420
|
if (cell.status === 'pending' || cell.status === 'selected') {
|
|
395
421
|
// Ran but never reached a terminal state (target threw before delivery).
|
|
396
|
-
|
|
422
|
+
// Persist a normalized reason so the operator sees a first-level cause,
|
|
423
|
+
// not a bare "target did not complete".
|
|
424
|
+
const error = cell.lastError ?? 'target did not complete';
|
|
425
|
+
this.database.slots.setCellStatus(slot.slotId, targetId, 'failed', error);
|
|
426
|
+
try {
|
|
427
|
+
this.database.slots.setCellTerminalReason(slot.slotId, targetId, 'internal_error', TargetOutcome_1.TERMINAL_REASON_MESSAGES.internal_error);
|
|
428
|
+
}
|
|
429
|
+
catch (reasonError) {
|
|
430
|
+
logger_1.logger.debug('Failed to persist terminal reason in rollup', {
|
|
431
|
+
slot: slot.slotId, target: targetId,
|
|
432
|
+
error: reasonError instanceof Error ? reasonError.message : String(reasonError),
|
|
433
|
+
});
|
|
434
|
+
}
|
|
397
435
|
}
|
|
398
436
|
}
|
|
399
437
|
const status = this.database.slots.deriveSlotStatus(slot.slotId);
|
|
@@ -403,6 +441,8 @@ class SlotCoordinator {
|
|
|
403
441
|
status: c.status,
|
|
404
442
|
workId: c.workId,
|
|
405
443
|
error: c.lastError,
|
|
444
|
+
terminalReasonCode: c.terminalReasonCode,
|
|
445
|
+
terminalReasonMessage: c.terminalReasonMessage,
|
|
406
446
|
}));
|
|
407
447
|
// Rolled up AFTER the slot row carries its terminal status/completed_at, so
|
|
408
448
|
// the outcome reports the durable timestamps rather than a fresh clock read.
|
|
@@ -450,6 +490,8 @@ class SlotCoordinator {
|
|
|
450
490
|
status: cell.status,
|
|
451
491
|
work_id: cell.workId,
|
|
452
492
|
error: cell.lastError,
|
|
493
|
+
terminal_reason_code: cell.terminalReasonCode,
|
|
494
|
+
terminal_reason_message: cell.terminalReasonMessage,
|
|
453
495
|
};
|
|
454
496
|
});
|
|
455
497
|
// A fully-submitted slot has no non-submitted cell at all, and reporting
|
|
@@ -253,4 +253,35 @@ export declare function mergeScanSummaries(first: CandidateScanSummary | null, s
|
|
|
253
253
|
export declare function classifyCandidateFailure(error: unknown, workId: string): CandidateFailure;
|
|
254
254
|
/** Hard job-level outage for a directly-thrown error, or null. */
|
|
255
255
|
export declare function classifyJobLevelOutage(error: unknown): JobLevelOutage | null;
|
|
256
|
+
/**
|
|
257
|
+
* Normalized terminal failure taxonomy (§terminal-reason).
|
|
258
|
+
*
|
|
259
|
+
* These codes are what operators actually see: the FIRST-LEVEL cause of a
|
|
260
|
+
* terminal cell. Recovery exhaustion is an execution state, never a root
|
|
261
|
+
* cause — a cell that exhausted its fallback stages because every candidate
|
|
262
|
+
* was a duplicate reports `duplicate_exhausted`, not `recovery_exhausted`.
|
|
263
|
+
* Queue waiting is not a terminal reason at all.
|
|
264
|
+
*
|
|
265
|
+
* Codes are kept deliberately small (no giant taxonomy): each maps to one
|
|
266
|
+
* real failure path in this codebase.
|
|
267
|
+
*/
|
|
268
|
+
export type TerminalReasonCode =
|
|
269
|
+
/** The candidate scan was empty — nothing was found in the day's ranked pool. */
|
|
270
|
+
'no_candidate'
|
|
271
|
+
/** Every candidate the scan saw was already delivered/pending for this target. */
|
|
272
|
+
| 'duplicate_exhausted'
|
|
273
|
+
/** Candidates existed but none passed the target's own filters. */
|
|
274
|
+
| 'filter_exhausted' | 'download_timeout' | 'download_failed' | 'metadata_failed' | 'rate_limited' | 'auth_failed' | 'remote_http_error' | 'delivery_failed' | 'telepost_rejected' | 'telegram_failed' | 'network_error' | 'execution_timeout' | 'configuration_error' | 'internal_error';
|
|
275
|
+
export interface TerminalReason {
|
|
276
|
+
code: TerminalReasonCode;
|
|
277
|
+
/** Business-language message an operator/reviewer reads directly. */
|
|
278
|
+
message: string;
|
|
279
|
+
}
|
|
280
|
+
/** User-facing business messages — never stack traces, paths, SQL or tokens. */
|
|
281
|
+
export declare const TERMINAL_REASON_MESSAGES: Record<TerminalReasonCode, string>;
|
|
282
|
+
/**
|
|
283
|
+
* Map a typed terminal TargetOutcome onto the normalized reason. Returns null
|
|
284
|
+
* for non-terminal outcomes (they have no terminal reason yet).
|
|
285
|
+
*/
|
|
286
|
+
export declare function terminalReasonFor(outcome: TargetOutcome): TerminalReason | null;
|
|
256
287
|
//# sourceMappingURL=TargetOutcome.d.ts.map
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.TERMINAL_REASON_MESSAGES = void 0;
|
|
3
4
|
exports.emptyCandidateScan = emptyCandidateScan;
|
|
4
5
|
exports.isTerminalOutcome = isTerminalOutcome;
|
|
5
6
|
exports.isSelectedAttempt = isSelectedAttempt;
|
|
@@ -10,6 +11,7 @@ exports.hasTransientFailure = hasTransientFailure;
|
|
|
10
11
|
exports.mergeScanSummaries = mergeScanSummaries;
|
|
11
12
|
exports.classifyCandidateFailure = classifyCandidateFailure;
|
|
12
13
|
exports.classifyJobLevelOutage = classifyJobLevelOutage;
|
|
14
|
+
exports.terminalReasonFor = terminalReasonFor;
|
|
13
15
|
/**
|
|
14
16
|
* A scan that attempted nothing. The real pipeline always reports its own scan;
|
|
15
17
|
* this is for callers/tests that have no candidate-level information, so they
|
|
@@ -224,4 +226,130 @@ function classifyJobLevelOutage(error) {
|
|
|
224
226
|
const failure = classifyCandidateFailure(error, '');
|
|
225
227
|
return failure.scope === 'job' ? failure.outage : null;
|
|
226
228
|
}
|
|
229
|
+
/** User-facing business messages — never stack traces, paths, SQL or tokens. */
|
|
230
|
+
exports.TERMINAL_REASON_MESSAGES = {
|
|
231
|
+
no_candidate: '没有找到合适的新作品',
|
|
232
|
+
duplicate_exhausted: '候选作品均已投稿过',
|
|
233
|
+
filter_exhausted: '没有符合筛选条件的新作品',
|
|
234
|
+
download_timeout: '图片下载超时',
|
|
235
|
+
download_failed: '图片下载失败',
|
|
236
|
+
metadata_failed: '作品信息获取失败',
|
|
237
|
+
rate_limited: 'Pixiv 请求频率受限',
|
|
238
|
+
auth_failed: 'Pixiv 登录已失效,需要重新登录',
|
|
239
|
+
remote_http_error: 'Pixiv 服务器返回错误',
|
|
240
|
+
delivery_failed: '投稿投递失败',
|
|
241
|
+
telepost_rejected: '投稿被接收端拒绝',
|
|
242
|
+
telegram_failed: 'Telegram 发送失败',
|
|
243
|
+
network_error: '网络异常',
|
|
244
|
+
execution_timeout: '执行超时',
|
|
245
|
+
configuration_error: '配置错误',
|
|
246
|
+
internal_error: '内部错误',
|
|
247
|
+
};
|
|
248
|
+
/**
|
|
249
|
+
* Map a typed terminal TargetOutcome onto the normalized reason. Returns null
|
|
250
|
+
* for non-terminal outcomes (they have no terminal reason yet).
|
|
251
|
+
*/
|
|
252
|
+
function terminalReasonFor(outcome) {
|
|
253
|
+
switch (outcome.kind) {
|
|
254
|
+
case 'submitted':
|
|
255
|
+
case 'stored':
|
|
256
|
+
case 'delivery_pending':
|
|
257
|
+
return null;
|
|
258
|
+
case 'no_candidate':
|
|
259
|
+
return reasonForNoCandidate(outcome);
|
|
260
|
+
case 'duplicate':
|
|
261
|
+
return {
|
|
262
|
+
code: 'duplicate_exhausted',
|
|
263
|
+
message: exports.TERMINAL_REASON_MESSAGES.duplicate_exhausted,
|
|
264
|
+
};
|
|
265
|
+
case 'failed':
|
|
266
|
+
return classifyFailedReason(outcome.error, outcome.scan);
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
/**
|
|
270
|
+
* Root cause for a `no_candidate` target, read from the scan's skip codes
|
|
271
|
+
* rather than from exhaustion bookkeeping: all duplicates ⇒
|
|
272
|
+
* `duplicate_exhausted`; otherwise the candidates were present but unusable
|
|
273
|
+
* (filter/deleted/denied/wrong media) ⇒ `filter_exhausted`; a scan that saw
|
|
274
|
+
* nothing at all ⇒ `no_candidate`.
|
|
275
|
+
*/
|
|
276
|
+
function reasonForNoCandidate(outcome) {
|
|
277
|
+
const skipped = outcome.scan?.skipped ?? [];
|
|
278
|
+
if (skipped.length > 0) {
|
|
279
|
+
const duplicates = skipped.filter((s) => s.code === 'duplicate').length;
|
|
280
|
+
if (duplicates === skipped.length) {
|
|
281
|
+
return { code: 'duplicate_exhausted', message: exports.TERMINAL_REASON_MESSAGES.duplicate_exhausted };
|
|
282
|
+
}
|
|
283
|
+
if (duplicates >= skipped.length / 2 || skipped.every((s) => s.code !== 'unavailable')) {
|
|
284
|
+
// Predominantly duplicates, or every skip was a hard work-level verdict:
|
|
285
|
+
// the pool contained works but none were usable for this target.
|
|
286
|
+
return { code: 'filter_exhausted', message: exports.TERMINAL_REASON_MESSAGES.filter_exhausted };
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
return { code: 'no_candidate', message: exports.TERMINAL_REASON_MESSAGES.no_candidate };
|
|
290
|
+
}
|
|
291
|
+
/**
|
|
292
|
+
* Classify a terminal `failed` reason from the error text + scan bookkeeping.
|
|
293
|
+
* Job-level outages from the scan win first (they describe the whole run), then
|
|
294
|
+
* message-shape heuristics; anything unclassifiable becomes `internal_error`
|
|
295
|
+
* rather than leaking raw error text to the review group.
|
|
296
|
+
*/
|
|
297
|
+
function classifyFailedReason(message, scan) {
|
|
298
|
+
const outage = scan?.outages?.[0];
|
|
299
|
+
if (outage === 'pixiv_auth_failure') {
|
|
300
|
+
return { code: 'auth_failed', message: exports.TERMINAL_REASON_MESSAGES.auth_failed };
|
|
301
|
+
}
|
|
302
|
+
if (outage === 'delivery_unavailable') {
|
|
303
|
+
return { code: 'delivery_failed', message: exports.TERMINAL_REASON_MESSAGES.delivery_failed };
|
|
304
|
+
}
|
|
305
|
+
if (outage === 'network_outage' || outage === 'database_unavailable') {
|
|
306
|
+
return { code: 'network_error', message: exports.TERMINAL_REASON_MESSAGES.network_error };
|
|
307
|
+
}
|
|
308
|
+
const text = String(message ?? '').slice(0, 600);
|
|
309
|
+
if (/\b429\b|rate ?limit/i.test(text)) {
|
|
310
|
+
return { code: 'rate_limited', message: exports.TERMINAL_REASON_MESSAGES.rate_limited };
|
|
311
|
+
}
|
|
312
|
+
if (/\b401\b|unauthorized|invalid grant|invalid refresh token|authentication failed|登录/i.test(text)) {
|
|
313
|
+
return { code: 'auth_failed', message: exports.TERMINAL_REASON_MESSAGES.auth_failed };
|
|
314
|
+
}
|
|
315
|
+
if (/timeout|timed ?out|timedout/i.test(text)) {
|
|
316
|
+
return /download|image|url|fetch/i.test(text)
|
|
317
|
+
? { code: 'download_timeout', message: exports.TERMINAL_REASON_MESSAGES.download_timeout }
|
|
318
|
+
: { code: 'execution_timeout', message: exports.TERMINAL_REASON_MESSAGES.execution_timeout };
|
|
319
|
+
}
|
|
320
|
+
if (/502|503|504|bad gateway|service unavailable|server error|5\d\d/i.test(text)) {
|
|
321
|
+
return { code: 'remote_http_error', message: exports.TERMINAL_REASON_MESSAGES.remote_http_error };
|
|
322
|
+
}
|
|
323
|
+
if (/telegram/i.test(text)) {
|
|
324
|
+
return { code: 'telegram_failed', message: exports.TERMINAL_REASON_MESSAGES.telegram_failed };
|
|
325
|
+
}
|
|
326
|
+
// Configuration faults are reported as such even when they surface inside a
|
|
327
|
+
// delivery/submission path ("delivery target not configured"): the operator
|
|
328
|
+
// fixes configuration, not the upstream.
|
|
329
|
+
if (/not configured|missing (?:config|configuration|setting)|invalid config/i.test(text)) {
|
|
330
|
+
return { code: 'configuration_error', message: exports.TERMINAL_REASON_MESSAGES.configuration_error };
|
|
331
|
+
}
|
|
332
|
+
// A candidate's metadata could not be gathered/parsed: this is its own root
|
|
333
|
+
// cause (config is fine; the upstream work was unreadable), so it must not
|
|
334
|
+
// fall through to internal_error or configuration_error.
|
|
335
|
+
if (/\bmetadata\b.{0,60}\b(?:fail|error|unable|invalid|parse|serialize|gather)\b|failed to (?:gather|parse|fetch|load|save)\b.{0,40}\bmetadata\b/i.test(text)) {
|
|
336
|
+
return { code: 'metadata_failed', message: exports.TERMINAL_REASON_MESSAGES.metadata_failed };
|
|
337
|
+
}
|
|
338
|
+
// TelePost (or any theme/review submission endpoint) explicitly rejected the
|
|
339
|
+
// work with a client-side / payload verdict. This is not a delivery outage
|
|
340
|
+
// (5xx/unreachable — handled above) nor a generic delivery failure.
|
|
341
|
+
if (/\btele[ _-]?post\b[^.\n]{0,60}\b(?:reject|invalid|4\d\d)\b/i.test(text)) {
|
|
342
|
+
return { code: 'telepost_rejected', message: exports.TERMINAL_REASON_MESSAGES.telepost_rejected };
|
|
343
|
+
}
|
|
344
|
+
if (/delivery|submit(?:t?ed)?|publish|post|outbox/i.test(text)) {
|
|
345
|
+
return { code: 'delivery_failed', message: exports.TERMINAL_REASON_MESSAGES.delivery_failed };
|
|
346
|
+
}
|
|
347
|
+
if (/econnrefused|econnreset|enotfound|etimedout|ehostunreach|enetunreach|socket hang up|network is unreachable|getaddrinfo/i.test(text)) {
|
|
348
|
+
return { code: 'network_error', message: exports.TERMINAL_REASON_MESSAGES.network_error };
|
|
349
|
+
}
|
|
350
|
+
if (/config|missing|invalid/i.test(text)) {
|
|
351
|
+
return { code: 'configuration_error', message: exports.TERMINAL_REASON_MESSAGES.configuration_error };
|
|
352
|
+
}
|
|
353
|
+
return { code: 'internal_error', message: exports.TERMINAL_REASON_MESSAGES.internal_error };
|
|
354
|
+
}
|
|
227
355
|
//# sourceMappingURL=TargetOutcome.js.map
|
|
@@ -108,6 +108,11 @@ class DatabaseMigration {
|
|
|
108
108
|
-- the terminal outcome can be reported back to the requester
|
|
109
109
|
-- without this service learning anything about the review.
|
|
110
110
|
correlation_id TEXT,
|
|
111
|
+
-- Manual recovery request UUID + policy preset ("重试一次/放宽条件重试").
|
|
112
|
+
-- A recovery slot re-runs the failed target(s) under a per-occurrence
|
|
113
|
+
-- policy and reports through the schedule-outcome channel.
|
|
114
|
+
recovery_request_id TEXT,
|
|
115
|
+
recovery_mode TEXT,
|
|
111
116
|
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
112
117
|
started_at DATETIME,
|
|
113
118
|
completed_at DATETIME,
|
|
@@ -126,6 +131,10 @@ class DatabaseMigration {
|
|
|
126
131
|
attempt_count INTEGER NOT NULL DEFAULT 0,
|
|
127
132
|
fallback_stage INTEGER NOT NULL DEFAULT 0,
|
|
128
133
|
last_error TEXT,
|
|
134
|
+
-- Normalized terminal failure reason (§terminal-reason): stable reason
|
|
135
|
+
-- code plus a user-facing business message, durable across restarts.
|
|
136
|
+
terminal_reason_code TEXT,
|
|
137
|
+
terminal_reason_message TEXT,
|
|
129
138
|
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
130
139
|
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
131
140
|
completed_at DATETIME,
|
|
@@ -239,6 +248,8 @@ class DatabaseMigration {
|
|
|
239
248
|
heartbeat_at: 'ALTER TABLE schedule_slots ADD COLUMN heartbeat_at INTEGER',
|
|
240
249
|
manual_request_id: 'ALTER TABLE schedule_slots ADD COLUMN manual_request_id TEXT',
|
|
241
250
|
correlation_id: 'ALTER TABLE schedule_slots ADD COLUMN correlation_id TEXT',
|
|
251
|
+
recovery_request_id: 'ALTER TABLE schedule_slots ADD COLUMN recovery_request_id TEXT',
|
|
252
|
+
recovery_mode: 'ALTER TABLE schedule_slots ADD COLUMN recovery_mode TEXT',
|
|
242
253
|
};
|
|
243
254
|
const columnAlters = [];
|
|
244
255
|
for (const [col, sql] of Object.entries(slotColumnMigrations)) {
|
|
@@ -249,6 +260,12 @@ class DatabaseMigration {
|
|
|
249
260
|
if (!itemCols.includes('fallback_stage')) {
|
|
250
261
|
columnAlters.push(`ALTER TABLE schedule_slot_items ADD COLUMN fallback_stage INTEGER NOT NULL DEFAULT 0`);
|
|
251
262
|
}
|
|
263
|
+
if (!itemCols.includes('terminal_reason_code')) {
|
|
264
|
+
columnAlters.push(`ALTER TABLE schedule_slot_items ADD COLUMN terminal_reason_code TEXT`);
|
|
265
|
+
}
|
|
266
|
+
if (!itemCols.includes('terminal_reason_message')) {
|
|
267
|
+
columnAlters.push(`ALTER TABLE schedule_slot_items ADD COLUMN terminal_reason_message TEXT`);
|
|
268
|
+
}
|
|
252
269
|
// Create indexes for better query performance
|
|
253
270
|
const indexes = [
|
|
254
271
|
`CREATE INDEX IF NOT EXISTS idx_downloads_pixiv_id_type ON downloads(pixiv_id, type)`,
|