pixivflow 2.19.1 → 2.19.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/ExecuteSlotCommand.js +1 -1
- package/dist/commands/SchedulerCommand.js +3 -1
- package/dist/commands/SchedulerRunOnceCommand.js +1 -1
- package/dist/commands/scheduler-runtime.d.ts +31 -2
- package/dist/commands/scheduler-runtime.js +63 -9
- package/dist/delivery/DeliveryLedgerPort.d.ts +14 -0
- package/dist/delivery/DeliveryLedgerPort.js +43 -0
- package/dist/delivery/OutboxWorker.d.ts +2 -0
- package/dist/delivery/OutboxWorker.js +25 -13
- package/dist/delivery/errorClass.d.ts +1 -1
- package/dist/delivery/errorClass.js +7 -0
- package/dist/download/DownloadManager.d.ts +5 -0
- package/dist/download/DownloadManager.js +12 -2
- package/dist/download/handlers/IllustrationTargetHandler.d.ts +27 -1
- package/dist/download/handlers/IllustrationTargetHandler.js +105 -7
- package/dist/download/handlers/NovelTargetHandler.d.ts +27 -1
- package/dist/download/handlers/NovelTargetHandler.js +107 -3
- package/dist/notification/NotificationPolicy.d.ts +11 -0
- package/dist/notification/NotificationPolicy.js +22 -5
- package/dist/package.json +1 -1
- package/dist/pixiv-client/TargetSearchRunner.js +24 -8
- package/dist/scheduler/SlotCoordinator.d.ts +74 -1
- package/dist/scheduler/SlotCoordinator.js +108 -1
- package/dist/scheduler/WorkIdentity.d.ts +49 -0
- package/dist/scheduler/WorkIdentity.js +31 -0
- package/dist/storage/repositories/DeliveryRepository.d.ts +9 -0
- package/dist/storage/repositories/DeliveryRepository.js +16 -0
- package/dist/storage/repositories/OutboxRepository.d.ts +15 -0
- package/dist/storage/repositories/OutboxRepository.js +31 -0
- package/dist/storage/repositories/SlotRepository.d.ts +27 -0
- package/dist/storage/repositories/SlotRepository.js +45 -0
- package/dist/topic/TopicPipeline.js +4 -0
- package/dist/utils/errors.d.ts +9 -0
- package/dist/utils/errors.js +14 -1
- package/dist/version.js +1 -1
- package/dist/webui/package.json +1 -1
- package/node_modules/@redtidev/pixiv-client/dist/transport/transport.d.ts +9 -0
- package/node_modules/@redtidev/pixiv-client/dist/transport/transport.d.ts.map +1 -1
- package/node_modules/@redtidev/pixiv-client/dist/transport/transport.js +66 -14
- package/node_modules/@redtidev/pixiv-client/dist/transport/transport.js.map +1 -1
- package/node_modules/@redtidev/pixiv-client/src/transport/__tests__/transport-cancellation.test.ts +257 -0
- package/node_modules/@redtidev/pixiv-client/src/transport/transport.ts +81 -18
- package/package.json +1 -1
|
@@ -135,7 +135,7 @@ class ExecuteSlotCommand extends Command_1.BaseCommand {
|
|
|
135
135
|
(0, runDiagnostics_1.recordRunError)(outcome.error);
|
|
136
136
|
},
|
|
137
137
|
...(excludedWorkIds ? { excludedWorkIds } : {}),
|
|
138
|
-
}), timeoutMs, () => runtime.cancelActive(`batch timeout after ${timeoutMs}ms
|
|
138
|
+
}), timeoutMs, () => runtime.cancelActive(`batch timeout after ${timeoutMs}ms`, 'timeout'), `slot ${slotId}`);
|
|
139
139
|
// Bounded drain: deliveries created by this run get one chance to confirm
|
|
140
140
|
// before we report. Rows that cannot finish now stay durable and are
|
|
141
141
|
// resolved by the control plane's reconciliation, never by a blind resend.
|
|
@@ -57,7 +57,9 @@ class SchedulerCommand extends Command_1.BaseCommand {
|
|
|
57
57
|
telemetry: {
|
|
58
58
|
beginRun: () => runtime.database.getOverviewStats().totalDownloads,
|
|
59
59
|
endRun: () => runtime.database.getOverviewStats().totalDownloads,
|
|
60
|
-
|
|
60
|
+
// `timeout`: the daemon stays alive, so the run's Slot must be taken
|
|
61
|
+
// terminal (the abort path does that) instead of staying recoverable.
|
|
62
|
+
requestCancel: (reason) => runtime.cancelActive(reason, 'timeout'),
|
|
61
63
|
},
|
|
62
64
|
});
|
|
63
65
|
const status = manager.start(runtime.config);
|
|
@@ -53,7 +53,7 @@ class SchedulerRunOnceCommand extends Command_1.BaseCommand {
|
|
|
53
53
|
// adhoc: a manual/operator run executes the download plan but never
|
|
54
54
|
// opens a scheduled Slot, so it can neither mark a scheduled
|
|
55
55
|
// occurrence complete nor be resumed as one. Explicit replacement.
|
|
56
|
-
runtime.runJob(runtime.config, plan, { adhoc: true, triggerSource: 'manual', onlyTarget: targetFilter }), timeoutMs, () => runtime.cancelActive(`run timeout after ${timeoutMs}ms
|
|
56
|
+
runtime.runJob(runtime.config, plan, { adhoc: true, triggerSource: 'manual', onlyTarget: targetFilter }), timeoutMs, () => runtime.cancelActive(`run timeout after ${timeoutMs}ms`, 'timeout'), `plan ${plan.id}`);
|
|
57
57
|
}
|
|
58
58
|
// Flush deliveries/notifications created by this one-shot before exit.
|
|
59
59
|
// A bounded drain: durable rows survive if they cannot finish now and the
|
|
@@ -51,6 +51,29 @@ export type ExecutionMode = (typeof EXECUTION_MODES)[number];
|
|
|
51
51
|
export declare function withDeliveryMode<T extends {
|
|
52
52
|
delivery?: unknown;
|
|
53
53
|
}>(targets: T[], mode: ExecutionMode | undefined): T[];
|
|
54
|
+
/**
|
|
55
|
+
* Who asked for the cancellation, which decides whether the abandoned Slot is
|
|
56
|
+
* allowed to stay recoverable.
|
|
57
|
+
*
|
|
58
|
+
* - `timeout`: a watchdog inside this process (scheduler timeout, run-once or
|
|
59
|
+
* execute-slot bound). The process stays up, so nothing will ever finish the
|
|
60
|
+
* Slot — it must be terminalised, or the recovery sweep re-dispatches the
|
|
61
|
+
* same occurrence on every tick forever.
|
|
62
|
+
* - `shutdown`: the process is going away. The Slot is deliberately left
|
|
63
|
+
* non-terminal so recovery resumes the same occurrence after restart; no
|
|
64
|
+
* worker survives to duplicate it.
|
|
65
|
+
*/
|
|
66
|
+
export type CancelOrigin = 'timeout' | 'shutdown';
|
|
67
|
+
/**
|
|
68
|
+
* Decide a scheduled run's Slot fate when `runAllTargets()` aborts abnormally
|
|
69
|
+
* instead of reporting per-target failures.
|
|
70
|
+
*
|
|
71
|
+
* Returning `false` leaves the Slot recoverable, which is only correct when the
|
|
72
|
+
* process is genuinely going away — because `recoverableSlots()` treats a
|
|
73
|
+
* released/NULL lease as "recorded but never claimed", so a live process that
|
|
74
|
+
* merely drops its lease is re-dispatched on every recovery tick, forever.
|
|
75
|
+
*/
|
|
76
|
+
export declare function shouldTerminaliseAbortedSlot(origin: CancelOrigin | null, slotAbandoned: boolean): boolean;
|
|
54
77
|
export interface SchedulerRuntime {
|
|
55
78
|
config: StandaloneConfig;
|
|
56
79
|
database: Database;
|
|
@@ -59,8 +82,14 @@ export interface SchedulerRuntime {
|
|
|
59
82
|
tokenMaintenance: ReturnType<typeof createTokenMaintenanceService>;
|
|
60
83
|
/** Run one schedule's enabled targets once (the same job the cron fires). */
|
|
61
84
|
runJob(snapshot: StandaloneConfig, schedule: ScheduleConfig, options?: RunJobOptions): Promise<void>;
|
|
62
|
-
/**
|
|
63
|
-
|
|
85
|
+
/**
|
|
86
|
+
* Cancel the in-flight download plan, if any holding a Slot.
|
|
87
|
+
*
|
|
88
|
+
* `origin` states whether this process is staying alive (`timeout`: the
|
|
89
|
+
* run is finished and its Slot must be terminalised) or going away
|
|
90
|
+
* (`shutdown`: leave the Slot non-terminal so recovery resumes it).
|
|
91
|
+
*/
|
|
92
|
+
cancelActive(reason: string, origin?: CancelOrigin): void;
|
|
64
93
|
/**
|
|
65
94
|
* The active run never settled after its timeout and drain window. Stop
|
|
66
95
|
* renewing its lease and take its Slot terminal so recovery cannot re-dispatch
|
|
@@ -43,6 +43,7 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
43
43
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
44
44
|
exports.EXECUTION_MODES = void 0;
|
|
45
45
|
exports.withDeliveryMode = withDeliveryMode;
|
|
46
|
+
exports.shouldTerminaliseAbortedSlot = shouldTerminaliseAbortedSlot;
|
|
46
47
|
exports.notifyScheduleFailure = notifyScheduleFailure;
|
|
47
48
|
exports.createSchedulerRuntime = createSchedulerRuntime;
|
|
48
49
|
exports.runWithTimeout = runWithTimeout;
|
|
@@ -56,6 +57,7 @@ const DeliveryDispatcher_1 = require("../delivery/DeliveryDispatcher");
|
|
|
56
57
|
const token_maintenance_1 = require("../utils/token-maintenance");
|
|
57
58
|
const schedules_1 = require("../scheduler/schedules");
|
|
58
59
|
const SlotCoordinator_1 = require("../scheduler/SlotCoordinator");
|
|
60
|
+
const DeliveryLedgerPort_1 = require("../delivery/DeliveryLedgerPort");
|
|
59
61
|
const OutboxWorker_1 = require("../delivery/OutboxWorker");
|
|
60
62
|
const LegacyOutboxMigration_1 = require("../delivery/LegacyOutboxMigration");
|
|
61
63
|
const NotificationPolicy_1 = require("../notification/NotificationPolicy");
|
|
@@ -74,6 +76,23 @@ function withDeliveryMode(targets, mode) {
|
|
|
74
76
|
return targets;
|
|
75
77
|
return targets.map((target) => (target.delivery ? { ...target, delivery: undefined } : target));
|
|
76
78
|
}
|
|
79
|
+
/**
|
|
80
|
+
* Decide a scheduled run's Slot fate when `runAllTargets()` aborts abnormally
|
|
81
|
+
* instead of reporting per-target failures.
|
|
82
|
+
*
|
|
83
|
+
* Returning `false` leaves the Slot recoverable, which is only correct when the
|
|
84
|
+
* process is genuinely going away — because `recoverableSlots()` treats a
|
|
85
|
+
* released/NULL lease as "recorded but never claimed", so a live process that
|
|
86
|
+
* merely drops its lease is re-dispatched on every recovery tick, forever.
|
|
87
|
+
*/
|
|
88
|
+
function shouldTerminaliseAbortedSlot(origin, slotAbandoned) {
|
|
89
|
+
// The abandon path already wrote a terminal `failed`; rolling up again would
|
|
90
|
+
// only overwrite it when the wedged job finally settles.
|
|
91
|
+
if (slotAbandoned)
|
|
92
|
+
return false;
|
|
93
|
+
// Shutdown is not a failure: recovery is meant to resume this occurrence.
|
|
94
|
+
return origin !== 'shutdown';
|
|
95
|
+
}
|
|
77
96
|
/**
|
|
78
97
|
* Open the pixivflow database with a startup integrity check. A structurally
|
|
79
98
|
* corrupt file (quick_check failure) is isolated aside and replaced with a
|
|
@@ -209,6 +228,13 @@ async function createSchedulerRuntime(configPathArg) {
|
|
|
209
228
|
* Set while a lease is held and cleared as soon as it is released.
|
|
210
229
|
*/
|
|
211
230
|
let activeLeaseHooks = null;
|
|
231
|
+
/**
|
|
232
|
+
* Why the in-flight run was cancelled, if it was. This is what lets the abort
|
|
233
|
+
* path tell a CRASH (nothing runs here — the process is gone) apart from an
|
|
234
|
+
* ABANDONMENT (this process is alive and will never touch the Slot again).
|
|
235
|
+
* `null` while no cancellation has been requested for the current run.
|
|
236
|
+
*/
|
|
237
|
+
let activeAbortOrigin = null;
|
|
212
238
|
// Independently-pumped durable outbox (content + notifications). Started in
|
|
213
239
|
// the long-running scheduler daemon; run-once drains explicitly before exit.
|
|
214
240
|
const deliveryDispatcher = new DeliveryDispatcher_1.DeliveryDispatcher(config.delivery, buildProxyUrl(config.network));
|
|
@@ -251,7 +277,7 @@ async function createSchedulerRuntime(configPathArg) {
|
|
|
251
277
|
});
|
|
252
278
|
return;
|
|
253
279
|
}
|
|
254
|
-
const coordinator = new SlotCoordinator_1.SlotCoordinator(database);
|
|
280
|
+
const coordinator = new SlotCoordinator_1.SlotCoordinator(database, (0, DeliveryLedgerPort_1.createDeliveryLedgerPort)(database));
|
|
255
281
|
// Ad-hoc/manual execution (run-once / explicit refetch) runs the download
|
|
256
282
|
// plan WITHOUT a scheduled Slot: it can never mark a scheduled occurrence
|
|
257
283
|
// complete or be resumed as one. Scheduled runs (cron/http/catchup) always
|
|
@@ -311,6 +337,9 @@ async function createSchedulerRuntime(configPathArg) {
|
|
|
311
337
|
// Only the process that actually owns the lease may report the slot as
|
|
312
338
|
// running; the accepting adapter records it as pending instead.
|
|
313
339
|
coordinator.markRunning(activeSlot.slotId);
|
|
340
|
+
// This run owns the Slot now: any cancellation recorded against a previous
|
|
341
|
+
// run must not decide how THIS run's abort is handled.
|
|
342
|
+
activeAbortOrigin = null;
|
|
314
343
|
let cancelled = false;
|
|
315
344
|
const heartbeat = setInterval(() => {
|
|
316
345
|
// A cancelled/timed-out run must stop renewing its lease. An infinitely
|
|
@@ -355,7 +384,14 @@ async function createSchedulerRuntime(configPathArg) {
|
|
|
355
384
|
// re-runs a finished cell — that is what prevents a second post). Membership
|
|
356
385
|
// comes from the materialized snapshot, so a config reload cannot add cells.
|
|
357
386
|
const pending = slotCtx ? coordinator.pendingTargets(slotCtx.slotId, targets) : targets.map((target) => ({ target, cell: null }));
|
|
358
|
-
|
|
387
|
+
const selected = onlyTarget ? pending.filter((p) => p.target.id === onlyTarget) : pending;
|
|
388
|
+
// Hand each cell its durable identity. Without this the handler cannot tell a
|
|
389
|
+
// first selection from a recovery, and would re-rank on resume — silently
|
|
390
|
+
// re-pointing the logical item at a different work than the one it owns.
|
|
391
|
+
const targetExecutionContexts = slotCtx
|
|
392
|
+
? coordinator.executionContextsFor(slotCtx.slotId, selected)
|
|
393
|
+
: undefined;
|
|
394
|
+
let runTargets = selected.map((p) => p.target);
|
|
359
395
|
if (slotCtx && runTargets.length === 0) {
|
|
360
396
|
logger_1.logger.info('All slot cells already complete', { slot: slotCtx.slotId });
|
|
361
397
|
coordinator.finish(slotCtx, schedule, targets);
|
|
@@ -396,6 +432,8 @@ async function createSchedulerRuntime(configPathArg) {
|
|
|
396
432
|
});
|
|
397
433
|
}
|
|
398
434
|
const downloadManager = new DownloadManager_1.DownloadManager(scopedConfig, pixivClient, database, fileService);
|
|
435
|
+
if (targetExecutionContexts)
|
|
436
|
+
downloadManager.setTargetExecutionContexts(targetExecutionContexts);
|
|
399
437
|
if (options.excludedWorkIds)
|
|
400
438
|
downloadManager.setProcessedWorkIds(options.excludedWorkIds);
|
|
401
439
|
activeDownloadManager = downloadManager;
|
|
@@ -451,9 +489,24 @@ async function createSchedulerRuntime(configPathArg) {
|
|
|
451
489
|
}
|
|
452
490
|
catch (error) {
|
|
453
491
|
if (!slotCtx || !(error instanceof Error) || !/^All \d+ target\(s\) failed\./.test(error.message)) {
|
|
454
|
-
// Abnormal abort
|
|
455
|
-
//
|
|
456
|
-
//
|
|
492
|
+
// Abnormal abort. Two very different situations land here, and treating
|
|
493
|
+
// them as one is what produced the production loop:
|
|
494
|
+
//
|
|
495
|
+
// - this process cancelled itself (scheduler timeout / watchdog) and is
|
|
496
|
+
// still alive. It will never touch the Slot again, so releasing the
|
|
497
|
+
// lease while leaving the status `running` makes the Slot look exactly
|
|
498
|
+
// like a crashed worker: `recoverableSlots()` matches a NULL lease on
|
|
499
|
+
// purpose (its "recorded but never claimed" case), so the recovery
|
|
500
|
+
// sweep re-dispatches the SAME occurrence on every tick, forever.
|
|
501
|
+
// Terminalise it instead: the occurrence is finished, and failed.
|
|
502
|
+
//
|
|
503
|
+
// - the process is going away (`shutdown`). Deliberately leave the Slot
|
|
504
|
+
// non-terminal so recovery resumes the same occurrence after restart.
|
|
505
|
+
if (slotCtx && shouldTerminaliseAbortedSlot(activeAbortOrigin, slotAbandoned)) {
|
|
506
|
+
coordinator.finish(slotCtx, schedule, targets);
|
|
507
|
+
}
|
|
508
|
+
// Hand the lease back either way: a terminal Slot cannot be re-dispatched,
|
|
509
|
+
// and a non-terminal one (shutdown) must not wait for its TTL to expire.
|
|
457
510
|
releaseLease?.();
|
|
458
511
|
throw error;
|
|
459
512
|
}
|
|
@@ -491,12 +544,13 @@ async function createSchedulerRuntime(configPathArg) {
|
|
|
491
544
|
});
|
|
492
545
|
logger_1.logger.info('='.repeat(60));
|
|
493
546
|
};
|
|
494
|
-
const cancelActive = (reason) => {
|
|
547
|
+
const cancelActive = (reason, origin = 'timeout') => {
|
|
548
|
+
activeAbortOrigin = origin;
|
|
495
549
|
activeDownloadManager?.cancel(reason);
|
|
496
550
|
// Scheduler timeout / process shutdown also stop the lease heartbeat. The run
|
|
497
551
|
// is on its way out (or about to be killed with the process), so it must not
|
|
498
|
-
// keep the slot locked while it unwinds.
|
|
499
|
-
//
|
|
552
|
+
// keep the slot locked while it unwinds. Whether the Slot is then terminalised
|
|
553
|
+
// or left recoverable is decided by `origin` in the abort path of runJob.
|
|
500
554
|
activeLeaseHooks?.stopHeartbeat();
|
|
501
555
|
};
|
|
502
556
|
const abandonActiveRun = (reason) => {
|
|
@@ -507,7 +561,7 @@ async function createSchedulerRuntime(configPathArg) {
|
|
|
507
561
|
};
|
|
508
562
|
const close = () => {
|
|
509
563
|
outboxWorker.stop();
|
|
510
|
-
cancelActive('process shutdown');
|
|
564
|
+
cancelActive('process shutdown', 'shutdown');
|
|
511
565
|
if (tokenMaintenance) {
|
|
512
566
|
tokenMaintenance.stop();
|
|
513
567
|
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { Database } from '../storage/Database';
|
|
2
|
+
import { SchedulerDeliveryPort } from '../scheduler/SlotCoordinator';
|
|
3
|
+
/**
|
|
4
|
+
* Answers the slot FSM's only delivery question — "does this cell still have an
|
|
5
|
+
* owner downstream?" — from the durable delivery ledger.
|
|
6
|
+
*
|
|
7
|
+
* The distinction matters after a crash: a `delivery_pending` cell whose intent
|
|
8
|
+
* is still live must be left alone (the OutboxWorker retries THAT work to a
|
|
9
|
+
* terminal ACK), while one whose intent is terminally failed must be failed
|
|
10
|
+
* explicitly. Both are read from committed rows, so the answer is the same
|
|
11
|
+
* before and after a restart — which is exactly what a recovery decision needs.
|
|
12
|
+
*/
|
|
13
|
+
export declare function createDeliveryLedgerPort(database: Database): SchedulerDeliveryPort;
|
|
14
|
+
//# sourceMappingURL=DeliveryLedgerPort.d.ts.map
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.createDeliveryLedgerPort = createDeliveryLedgerPort;
|
|
4
|
+
/**
|
|
5
|
+
* Answers the slot FSM's only delivery question — "does this cell still have an
|
|
6
|
+
* owner downstream?" — from the durable delivery ledger.
|
|
7
|
+
*
|
|
8
|
+
* The distinction matters after a crash: a `delivery_pending` cell whose intent
|
|
9
|
+
* is still live must be left alone (the OutboxWorker retries THAT work to a
|
|
10
|
+
* terminal ACK), while one whose intent is terminally failed must be failed
|
|
11
|
+
* explicitly. Both are read from committed rows, so the answer is the same
|
|
12
|
+
* before and after a restart — which is exactly what a recovery decision needs.
|
|
13
|
+
*/
|
|
14
|
+
function createDeliveryLedgerPort(database) {
|
|
15
|
+
return {
|
|
16
|
+
stateFor({ deliveryTarget, slotId, targetId }) {
|
|
17
|
+
const rows = database.deliveries.listForCell(deliveryTarget, slotId, targetId);
|
|
18
|
+
if (rows.length === 0)
|
|
19
|
+
return { kind: 'unknown' };
|
|
20
|
+
const confirmed = rows.find((r) => r.status === 'delivered' || r.status === 'duplicate');
|
|
21
|
+
if (confirmed) {
|
|
22
|
+
return { kind: 'confirmed', workId: confirmed.pixivId, workType: confirmed.workType };
|
|
23
|
+
}
|
|
24
|
+
const pending = rows.filter((r) => r.status === 'pending');
|
|
25
|
+
// Still actionable = the outbox is going to try again. Its retry budget is
|
|
26
|
+
// the only thing that may converge this delivery, so keep hands off.
|
|
27
|
+
if (pending.some((r) => database.outbox.hasActionableDelivery(r.id))) {
|
|
28
|
+
return { kind: 'live' };
|
|
29
|
+
}
|
|
30
|
+
// Either the outbox exhausted its retries (dead/failed) or the intent was
|
|
31
|
+
// never queued. Nobody will converge it, and re-running selection would
|
|
32
|
+
// re-point the logical item at a different work — so fail it as-is.
|
|
33
|
+
if (pending.length > 0 || rows.some((r) => r.status === 'failed')) {
|
|
34
|
+
return {
|
|
35
|
+
kind: 'lost',
|
|
36
|
+
reason: 'delivery intent reached a terminal failure without an ACK; failing the cell instead of re-selecting a different work',
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
return { kind: 'unknown' };
|
|
40
|
+
},
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
//# sourceMappingURL=DeliveryLedgerPort.js.map
|
|
@@ -78,6 +78,8 @@ export declare class OutboxWorker {
|
|
|
78
78
|
private recordMediaFallback;
|
|
79
79
|
private recordEvent;
|
|
80
80
|
private process;
|
|
81
|
+
/** Terminal delivery failure: audit it, release the delivery, notify the hooks. */
|
|
82
|
+
private deadLetter;
|
|
81
83
|
private handleDeliveryAck;
|
|
82
84
|
private cleanup;
|
|
83
85
|
}
|
|
@@ -245,24 +245,21 @@ class OutboxWorker {
|
|
|
245
245
|
return 'done';
|
|
246
246
|
}
|
|
247
247
|
catch (error) {
|
|
248
|
-
// Transport-level error (fetch threw) or ack-classified failure: retryable.
|
|
249
248
|
const message = (0, redact_1.redactError)(error).slice(0, 1000);
|
|
250
249
|
const { errorClass } = (0, errorClass_1.classifyError)(error);
|
|
250
|
+
// A local configuration error is deterministic: every attempt re-reads the
|
|
251
|
+
// same config, so retrying only parks the row in `retry_wait` until its
|
|
252
|
+
// attempt budget runs out. Dead-letter it now so it is visible instead.
|
|
253
|
+
if (errorClass === 'configuration_error') {
|
|
254
|
+
this.database.outbox.markDead(row.id, message);
|
|
255
|
+
this.deadLetter(row, `${message} (configuration error)`, errorClass);
|
|
256
|
+
return 'dead';
|
|
257
|
+
}
|
|
258
|
+
// Transport-level error (fetch threw) or ack-classified failure: retryable.
|
|
251
259
|
const delay = backoffDelayMs(row.attempts + 1, this.retryBaseMs, this.retryMaxMs);
|
|
252
260
|
const status = this.database.outbox.markRetry(row.id, Date.now() + delay, message);
|
|
253
261
|
if (status === 'dead') {
|
|
254
|
-
|
|
255
|
-
this.recordEvent(row, {
|
|
256
|
-
event: 'outbox.dead',
|
|
257
|
-
errorClass,
|
|
258
|
-
retryable: false,
|
|
259
|
-
countsAsAttempt: 1,
|
|
260
|
-
detail: { attempt: row.attempts + 1 },
|
|
261
|
-
});
|
|
262
|
-
this.onDead?.(row, message);
|
|
263
|
-
if (row.deliveryId) {
|
|
264
|
-
this.database.deliveries.recordAck(row.deliveryId, { status: 'failed', error: message });
|
|
265
|
-
}
|
|
262
|
+
this.deadLetter(row, message, errorClass);
|
|
266
263
|
}
|
|
267
264
|
else {
|
|
268
265
|
logger_1.logger.warn('Outbox item will retry', { outboxId: row.id, kind: row.kind, attempt: row.attempts + 1, delayMs: delay });
|
|
@@ -277,6 +274,21 @@ class OutboxWorker {
|
|
|
277
274
|
return status;
|
|
278
275
|
}
|
|
279
276
|
}
|
|
277
|
+
/** Terminal delivery failure: audit it, release the delivery, notify the hooks. */
|
|
278
|
+
deadLetter(row, message, errorClass) {
|
|
279
|
+
logger_1.logger.error('Outbox item dead', { outboxId: row.id, kind: row.kind, errorClass, error: message });
|
|
280
|
+
this.recordEvent(row, {
|
|
281
|
+
event: 'outbox.dead',
|
|
282
|
+
errorClass,
|
|
283
|
+
retryable: false,
|
|
284
|
+
countsAsAttempt: 1,
|
|
285
|
+
detail: { attempt: row.attempts + 1 },
|
|
286
|
+
});
|
|
287
|
+
this.onDead?.(row, message);
|
|
288
|
+
if (row.deliveryId) {
|
|
289
|
+
this.database.deliveries.recordAck(row.deliveryId, { status: 'failed', error: message });
|
|
290
|
+
}
|
|
291
|
+
}
|
|
280
292
|
async handleDeliveryAck(row, ack, payload) {
|
|
281
293
|
if (!row.deliveryId)
|
|
282
294
|
return;
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Tiny pure error classification for delivery attempts.
|
|
3
3
|
* Maps a thrown error / HTTP status to a stable audit category.
|
|
4
4
|
*/
|
|
5
|
-
export type DeliveryErrorClass = 'dependency_not_ready' | 'network_timeout' | 'rate_limited' | 'remote_5xx' | 'remote_4xx' | 'invalid_payload' | 'telegram_send_failed' | 'duplicate' | 'internal_error';
|
|
5
|
+
export type DeliveryErrorClass = 'dependency_not_ready' | 'network_timeout' | 'rate_limited' | 'remote_5xx' | 'remote_4xx' | 'invalid_payload' | 'telegram_send_failed' | 'configuration_error' | 'duplicate' | 'internal_error';
|
|
6
6
|
export interface ErrorClassification {
|
|
7
7
|
errorClass: DeliveryErrorClass;
|
|
8
8
|
retryable: boolean;
|
|
@@ -4,6 +4,13 @@ exports.classifyError = classifyError;
|
|
|
4
4
|
/** Classify one failed delivery attempt from its thrown error and/or HTTP status. */
|
|
5
5
|
function classifyError(error, status) {
|
|
6
6
|
const message = error instanceof Error ? error.message : String(error ?? '');
|
|
7
|
+
// A local ConfigError is thrown before any network call and is deterministic:
|
|
8
|
+
// retrying re-reads the exact same config, so the attempt budget only parks the
|
|
9
|
+
// row in `retry_wait` for hours. Terminal for the outbox.
|
|
10
|
+
if (error?.name === 'ConfigError' ||
|
|
11
|
+
/does not configure|unsupported delivery target/i.test(message)) {
|
|
12
|
+
return { errorClass: 'configuration_error', retryable: false };
|
|
13
|
+
}
|
|
7
14
|
// Explicit HTTP status wins when present.
|
|
8
15
|
if (status === 429)
|
|
9
16
|
return { errorClass: 'rate_limited', retryable: true };
|
|
@@ -5,6 +5,7 @@ import { IDatabase } from '../interfaces/IDatabase';
|
|
|
5
5
|
import { IFileService } from '../interfaces/IFileService';
|
|
6
6
|
import { DeliveryService } from '../delivery/DeliveryService';
|
|
7
7
|
import { TargetOutcome } from '../scheduler/TargetOutcome';
|
|
8
|
+
import { TargetExecutionContext } from '../scheduler/WorkIdentity';
|
|
8
9
|
/**
|
|
9
10
|
* Download Manager with Concurrency Control
|
|
10
11
|
*
|
|
@@ -76,6 +77,10 @@ export declare class DownloadManager implements IDownloadManager {
|
|
|
76
77
|
slotName: string;
|
|
77
78
|
slotDate: string;
|
|
78
79
|
};
|
|
80
|
+
/** Per-cell work identity for THIS run, keyed by target id (scheduled runs only). */
|
|
81
|
+
private targetExecutionContexts;
|
|
82
|
+
/** Publish the durable cell identity each handler must honour (scheduled runs). */
|
|
83
|
+
setTargetExecutionContexts(contexts: Map<string, TargetExecutionContext>): void;
|
|
79
84
|
/**
|
|
80
85
|
* Request cooperative cancellation of the current run. In-flight item
|
|
81
86
|
* finishes; no further targets/items are started. runAllTargets() will
|
|
@@ -86,6 +86,12 @@ class DownloadManager {
|
|
|
86
86
|
}
|
|
87
87
|
/** Slot context propagated to delivery templates/outcomes for scheduled runs. */
|
|
88
88
|
slotContext;
|
|
89
|
+
/** Per-cell work identity for THIS run, keyed by target id (scheduled runs only). */
|
|
90
|
+
targetExecutionContexts = new Map();
|
|
91
|
+
/** Publish the durable cell identity each handler must honour (scheduled runs). */
|
|
92
|
+
setTargetExecutionContexts(contexts) {
|
|
93
|
+
this.targetExecutionContexts = contexts;
|
|
94
|
+
}
|
|
89
95
|
/**
|
|
90
96
|
* Request cooperative cancellation of the current run. In-flight item
|
|
91
97
|
* finishes; no further targets/items are started. runAllTargets() will
|
|
@@ -216,11 +222,15 @@ class DownloadManager {
|
|
|
216
222
|
}
|
|
217
223
|
}
|
|
218
224
|
async dispatchTarget(target) {
|
|
225
|
+
// The cell identity must reach the handler or it cannot tell a first
|
|
226
|
+
// selection from a recovery: it would re-rank and silently re-point the
|
|
227
|
+
// logical item at a different work.
|
|
228
|
+
const execution = target.id ? this.targetExecutionContexts.get(target.id) : undefined;
|
|
219
229
|
switch (target.type) {
|
|
220
230
|
case 'illustration':
|
|
221
|
-
return await this.illustrationHandler.handle(target);
|
|
231
|
+
return await this.illustrationHandler.handle(target, execution);
|
|
222
232
|
case 'novel':
|
|
223
|
-
return await this.novelHandler.handle(target);
|
|
233
|
+
return await this.novelHandler.handle(target, execution);
|
|
224
234
|
default:
|
|
225
235
|
logger_1.logger.warn(`Unsupported target type ${target.type}`);
|
|
226
236
|
return { kind: 'failed', retryable: false, error: `unsupported target type ${target.type}` };
|
|
@@ -6,6 +6,7 @@ import { IllustrationDownloader } from '../IllustrationDownloader';
|
|
|
6
6
|
import { DownloadPipeline } from '../pipeline/DownloadPipeline';
|
|
7
7
|
import { DeliveryService } from '../../delivery/DeliveryService';
|
|
8
8
|
import { TargetOutcome } from '../../scheduler/TargetOutcome';
|
|
9
|
+
import { TargetExecutionContext } from '../../scheduler/WorkIdentity';
|
|
9
10
|
import type { TopicPipelineFactory } from '../../topic/createTopicPipeline';
|
|
10
11
|
export declare class IllustrationTargetHandler {
|
|
11
12
|
private readonly client;
|
|
@@ -17,8 +18,14 @@ export declare class IllustrationTargetHandler {
|
|
|
17
18
|
private readonly deliveryService?;
|
|
18
19
|
/** Outcomes produced during this handle() call (deliveries + terminal non-matches). */
|
|
19
20
|
private outcomes;
|
|
21
|
+
/**
|
|
22
|
+
* Cell identity for this handle() call. Set only for a single-work cell of a
|
|
23
|
+
* scheduled occurrence; null for ad-hoc runs and N-works-per-run targets, which
|
|
24
|
+
* have no single (slotId,targetId) -> workId identity to honour.
|
|
25
|
+
*/
|
|
26
|
+
private execution;
|
|
20
27
|
constructor(client: IPixivClient, database: IDatabase, rankingService: RankingService, illustrationDownloader: IllustrationDownloader, pipeline: DownloadPipeline, topicPipelineFactory?: TopicPipelineFactory | undefined, deliveryService?: DeliveryService | undefined);
|
|
21
|
-
handle(target: TargetConfig): Promise<TargetOutcome>;
|
|
28
|
+
handle(target: TargetConfig, execution?: TargetExecutionContext): Promise<TargetOutcome>;
|
|
22
29
|
/** Reduce the outcomes collected while processing one target to one cell result. */
|
|
23
30
|
private summarize;
|
|
24
31
|
private classifyError;
|
|
@@ -35,6 +42,25 @@ export declare class IllustrationTargetHandler {
|
|
|
35
42
|
private handleUserIllustrations;
|
|
36
43
|
private sortByPopularityAndLog;
|
|
37
44
|
private logError;
|
|
45
|
+
/**
|
|
46
|
+
* Continue the work this cell ALREADY owns. Selection is deliberately skipped:
|
|
47
|
+
* no search, no ranking, no topic expansion, no backfill pool, and no
|
|
48
|
+
* "already downloaded / already delivered" exclusion — the cell's own work must
|
|
49
|
+
* never be filtered out of its own recovery.
|
|
50
|
+
*
|
|
51
|
+
* A locked work that is permanently gone is a terminal failure of THIS logical
|
|
52
|
+
* item (`LOCKED_WORK_UNAVAILABLE`). It is never silently replaced by another
|
|
53
|
+
* candidate; that would mutate the item's identity behind the operator's back.
|
|
54
|
+
*/
|
|
55
|
+
private recoverLockedWork;
|
|
56
|
+
/**
|
|
57
|
+
* Process ONE candidate work for this cell.
|
|
58
|
+
*
|
|
59
|
+
* The cell is bound to `illust` BEFORE any side effect: it is the binding, not
|
|
60
|
+
* the candidate list, that decides what a later recovery resumes. The binding
|
|
61
|
+
* is only rolled back when the attempt produced no artifact at all, so in-run
|
|
62
|
+
* backfill still works for a cell that has committed to nothing.
|
|
63
|
+
*/
|
|
38
64
|
private downloadAndDeliver;
|
|
39
65
|
/**
|
|
40
66
|
* Turn a downloaded artifact into the target's business outcome. In cache
|
|
@@ -5,6 +5,7 @@ const logger_1 = require("../../logger");
|
|
|
5
5
|
const pixiv_date_utils_1 = require("../../utils/pixiv-date-utils");
|
|
6
6
|
const errors_1 = require("../../utils/errors");
|
|
7
7
|
const pixiv_utils_1 = require("../../utils/pixiv-utils");
|
|
8
|
+
const WorkIdentity_1 = require("../../scheduler/WorkIdentity");
|
|
8
9
|
const target_label_1 = require("../../utils/target-label");
|
|
9
10
|
class IllustrationTargetHandler {
|
|
10
11
|
client;
|
|
@@ -16,6 +17,12 @@ class IllustrationTargetHandler {
|
|
|
16
17
|
deliveryService;
|
|
17
18
|
/** Outcomes produced during this handle() call (deliveries + terminal non-matches). */
|
|
18
19
|
outcomes = [];
|
|
20
|
+
/**
|
|
21
|
+
* Cell identity for this handle() call. Set only for a single-work cell of a
|
|
22
|
+
* scheduled occurrence; null for ad-hoc runs and N-works-per-run targets, which
|
|
23
|
+
* have no single (slotId,targetId) -> workId identity to honour.
|
|
24
|
+
*/
|
|
25
|
+
execution = null;
|
|
19
26
|
constructor(client, database, rankingService, illustrationDownloader, pipeline, topicPipelineFactory, deliveryService) {
|
|
20
27
|
this.client = client;
|
|
21
28
|
this.database = database;
|
|
@@ -25,8 +32,17 @@ class IllustrationTargetHandler {
|
|
|
25
32
|
this.topicPipelineFactory = topicPipelineFactory;
|
|
26
33
|
this.deliveryService = deliveryService;
|
|
27
34
|
}
|
|
28
|
-
async handle(target) {
|
|
35
|
+
async handle(target, execution) {
|
|
29
36
|
this.outcomes = [];
|
|
37
|
+
this.execution = execution && (0, WorkIdentity_1.isSingleWorkCell)(target) ? execution : null;
|
|
38
|
+
// A cell that already owns a work is in RECOVERY, not in a new selection.
|
|
39
|
+
// Crash/shutdown recovery is not an intentional second run: running the
|
|
40
|
+
// candidate pipeline here would re-rank and could bind this logical item to a
|
|
41
|
+
// different work than the one it already committed to.
|
|
42
|
+
if (this.execution?.lockedWorkId) {
|
|
43
|
+
await this.recoverLockedWork(target, this.execution.lockedWorkId);
|
|
44
|
+
return this.summarize(target);
|
|
45
|
+
}
|
|
30
46
|
if (target.illustId) {
|
|
31
47
|
await this.handleSingleIllustration(target);
|
|
32
48
|
return this.summarize(target);
|
|
@@ -361,14 +377,96 @@ class IllustrationTargetHandler {
|
|
|
361
377
|
stack: error instanceof Error ? error.stack : undefined,
|
|
362
378
|
});
|
|
363
379
|
}
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
380
|
+
/**
|
|
381
|
+
* Continue the work this cell ALREADY owns. Selection is deliberately skipped:
|
|
382
|
+
* no search, no ranking, no topic expansion, no backfill pool, and no
|
|
383
|
+
* "already downloaded / already delivered" exclusion — the cell's own work must
|
|
384
|
+
* never be filtered out of its own recovery.
|
|
385
|
+
*
|
|
386
|
+
* A locked work that is permanently gone is a terminal failure of THIS logical
|
|
387
|
+
* item (`LOCKED_WORK_UNAVAILABLE`). It is never silently replaced by another
|
|
388
|
+
* candidate; that would mutate the item's identity behind the operator's back.
|
|
389
|
+
*/
|
|
390
|
+
async recoverLockedWork(target, lockedWorkId) {
|
|
391
|
+
const displayTag = (0, target_label_1.getTargetLabel)(target);
|
|
392
|
+
logger_1.logger.info(`Recovering locked work ${lockedWorkId} for ${displayTag}; candidate selection is skipped`, {
|
|
393
|
+
slotId: this.execution?.slotId,
|
|
394
|
+
targetId: this.execution?.targetId,
|
|
395
|
+
lockedWorkId,
|
|
369
396
|
});
|
|
370
|
-
|
|
397
|
+
const illustId = Number(lockedWorkId);
|
|
398
|
+
if (!Number.isFinite(illustId)) {
|
|
399
|
+
this.outcomes.push({
|
|
400
|
+
kind: 'failed',
|
|
401
|
+
retryable: false,
|
|
402
|
+
error: `LOCKED_WORK_UNAVAILABLE: cell work id "${lockedWorkId}" is not a valid illustration id`,
|
|
403
|
+
});
|
|
371
404
|
return;
|
|
405
|
+
}
|
|
406
|
+
try {
|
|
407
|
+
const detail = await this.client.getIllustration(illustId);
|
|
408
|
+
await this.downloadAndDeliver(detail, displayTag, target);
|
|
409
|
+
}
|
|
410
|
+
catch (error) {
|
|
411
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
412
|
+
const retryable = (0, errors_1.isRetryableNetworkError)(error);
|
|
413
|
+
this.logError(error, `Failed to recover locked illustration ${lockedWorkId}`);
|
|
414
|
+
this.outcomes.push({
|
|
415
|
+
kind: 'failed',
|
|
416
|
+
retryable,
|
|
417
|
+
error: retryable
|
|
418
|
+
? `locked work ${lockedWorkId} could not be fetched (will retry the SAME work): ${message}`
|
|
419
|
+
: `LOCKED_WORK_UNAVAILABLE: locked work ${lockedWorkId} is no longer fetchable: ${message}`,
|
|
420
|
+
});
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
/**
|
|
424
|
+
* Process ONE candidate work for this cell.
|
|
425
|
+
*
|
|
426
|
+
* The cell is bound to `illust` BEFORE any side effect: it is the binding, not
|
|
427
|
+
* the candidate list, that decides what a later recovery resumes. The binding
|
|
428
|
+
* is only rolled back when the attempt produced no artifact at all, so in-run
|
|
429
|
+
* backfill still works for a cell that has committed to nothing.
|
|
430
|
+
*/
|
|
431
|
+
async downloadAndDeliver(illust, tag, target) {
|
|
432
|
+
const execution = this.execution;
|
|
433
|
+
const workId = String(illust.id);
|
|
434
|
+
// Recovery continues a work the cell already bound; it must never be released.
|
|
435
|
+
const recovering = Boolean(execution?.lockedWorkId);
|
|
436
|
+
if (execution && !recovering) {
|
|
437
|
+
const binding = execution.bind(workId, 'illustration');
|
|
438
|
+
if (!binding.won) {
|
|
439
|
+
// Another writer elected a different work for this cell. First selection
|
|
440
|
+
// is authoritative: never process a work the cell does not own.
|
|
441
|
+
logger_1.logger.warn(`Cell is bound to work ${binding.workId}; declining to select ${workId}`, {
|
|
442
|
+
slotId: execution.slotId,
|
|
443
|
+
targetId: execution.targetId,
|
|
444
|
+
boundWorkId: binding.workId,
|
|
445
|
+
});
|
|
446
|
+
return;
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
let artifact = null;
|
|
450
|
+
try {
|
|
451
|
+
artifact = await this.illustrationDownloader.downloadIllustration(illust, tag, {
|
|
452
|
+
aiMetadataCheck: target.aiMetadataCheck === true,
|
|
453
|
+
maxPageCount: target.maxPageCount,
|
|
454
|
+
includeDeliveryPreviews: Boolean(target.storageMode === 'cache' && target.delivery?.target?.trim()),
|
|
455
|
+
});
|
|
456
|
+
}
|
|
457
|
+
catch (error) {
|
|
458
|
+
// Nothing was persisted, so the cell may still pick another candidate.
|
|
459
|
+
if (execution && !recovering)
|
|
460
|
+
execution.release(workId);
|
|
461
|
+
throw error;
|
|
462
|
+
}
|
|
463
|
+
if (!artifact) {
|
|
464
|
+
if (execution && !recovering)
|
|
465
|
+
execution.release(workId);
|
|
466
|
+
return;
|
|
467
|
+
}
|
|
468
|
+
// Committed: the artifact is durable and this work now defines the cell. Even
|
|
469
|
+
// if enqueue throws below, recovery must resume THIS work — never release.
|
|
372
470
|
this.recordArtifactOutcome(artifact, target);
|
|
373
471
|
}
|
|
374
472
|
/**
|