pixivflow 2.19.2 → 2.19.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/scheduler-runtime.js +16 -17
- package/dist/delivery/DeliveryAck.d.ts +14 -0
- package/dist/delivery/DeliveryAck.js +18 -0
- package/dist/delivery/DeliveryLedgerPort.d.ts +14 -0
- package/dist/delivery/DeliveryLedgerPort.js +43 -0
- package/dist/delivery/OutboxWorker.js +13 -0
- package/dist/delivery/settleDeliveryTerminal.d.ts +25 -0
- package/dist/delivery/settleDeliveryTerminal.js +40 -0
- package/dist/download/DownloadManager.d.ts +5 -0
- package/dist/download/DownloadManager.js +12 -2
- package/dist/download/handlers/IllustrationTargetHandler.d.ts +27 -1
- package/dist/download/handlers/IllustrationTargetHandler.js +105 -7
- package/dist/download/handlers/NovelTargetHandler.d.ts +27 -1
- package/dist/download/handlers/NovelTargetHandler.js +107 -3
- package/dist/package.json +1 -1
- package/dist/pixiv-client/TargetSearchRunner.js +24 -8
- package/dist/scheduler/SlotCoordinator.d.ts +74 -1
- package/dist/scheduler/SlotCoordinator.js +108 -1
- package/dist/scheduler/WorkIdentity.d.ts +49 -0
- package/dist/scheduler/WorkIdentity.js +31 -0
- package/dist/storage/repositories/DeliveryRepository.d.ts +9 -0
- package/dist/storage/repositories/DeliveryRepository.js +16 -0
- package/dist/storage/repositories/OutboxRepository.d.ts +9 -0
- package/dist/storage/repositories/OutboxRepository.js +17 -0
- package/dist/storage/repositories/SlotRepository.d.ts +27 -0
- package/dist/storage/repositories/SlotRepository.js +45 -0
- package/dist/topic/TopicPipeline.js +4 -0
- package/dist/utils/errors.d.ts +9 -0
- package/dist/utils/errors.js +14 -1
- package/dist/version.js +1 -1
- package/dist/webui/package.json +1 -1
- package/package.json +1 -1
|
@@ -57,7 +57,9 @@ const DeliveryDispatcher_1 = require("../delivery/DeliveryDispatcher");
|
|
|
57
57
|
const token_maintenance_1 = require("../utils/token-maintenance");
|
|
58
58
|
const schedules_1 = require("../scheduler/schedules");
|
|
59
59
|
const SlotCoordinator_1 = require("../scheduler/SlotCoordinator");
|
|
60
|
+
const DeliveryLedgerPort_1 = require("../delivery/DeliveryLedgerPort");
|
|
60
61
|
const OutboxWorker_1 = require("../delivery/OutboxWorker");
|
|
62
|
+
const settleDeliveryTerminal_1 = require("../delivery/settleDeliveryTerminal");
|
|
61
63
|
const LegacyOutboxMigration_1 = require("../delivery/LegacyOutboxMigration");
|
|
62
64
|
const NotificationPolicy_1 = require("../notification/NotificationPolicy");
|
|
63
65
|
const node_crypto_1 = require("node:crypto");
|
|
@@ -240,22 +242,10 @@ async function createSchedulerRuntime(configPathArg) {
|
|
|
240
242
|
const outboxWorker = new OutboxWorker_1.OutboxWorker(database, deliveryDispatcher, {
|
|
241
243
|
retryBaseMs: config.delivery?.outboxRetryBaseMs,
|
|
242
244
|
retryMaxMs: config.delivery?.outboxRetryMaxMs,
|
|
243
|
-
// A confirmed ACK
|
|
245
|
+
// A confirmed ACK settles the owning Slot cell (submitted / duplicate /
|
|
246
|
+
// failed); see settleDeliveryTerminal for the invariant it enforces.
|
|
244
247
|
onDeliveryTerminal: (deliveryId, ack) => {
|
|
245
|
-
|
|
246
|
-
if (!row || !row.slotId || !row.targetId)
|
|
247
|
-
return;
|
|
248
|
-
if (ack.kind === 'duplicate_existing') {
|
|
249
|
-
const coord = new SlotCoordinator_1.SlotCoordinator(database);
|
|
250
|
-
coord.applyOutcome(row.slotId, row.targetId, {
|
|
251
|
-
kind: 'duplicate',
|
|
252
|
-
workId: row.pixivId,
|
|
253
|
-
reason: 'downstream attested historical duplicate',
|
|
254
|
-
});
|
|
255
|
-
return;
|
|
256
|
-
}
|
|
257
|
-
const coord = new SlotCoordinator_1.SlotCoordinator(database);
|
|
258
|
-
coord.markDelivered(row.slotId, row.targetId, row.pixivId, row.workType);
|
|
248
|
+
(0, settleDeliveryTerminal_1.settleDeliveryTerminal)(database, deliveryId, ack);
|
|
259
249
|
},
|
|
260
250
|
});
|
|
261
251
|
const notificationPolicy = new NotificationPolicy_1.NotificationPolicy(database, config);
|
|
@@ -276,7 +266,7 @@ async function createSchedulerRuntime(configPathArg) {
|
|
|
276
266
|
});
|
|
277
267
|
return;
|
|
278
268
|
}
|
|
279
|
-
const coordinator = new SlotCoordinator_1.SlotCoordinator(database);
|
|
269
|
+
const coordinator = new SlotCoordinator_1.SlotCoordinator(database, (0, DeliveryLedgerPort_1.createDeliveryLedgerPort)(database));
|
|
280
270
|
// Ad-hoc/manual execution (run-once / explicit refetch) runs the download
|
|
281
271
|
// plan WITHOUT a scheduled Slot: it can never mark a scheduled occurrence
|
|
282
272
|
// complete or be resumed as one. Scheduled runs (cron/http/catchup) always
|
|
@@ -383,7 +373,14 @@ async function createSchedulerRuntime(configPathArg) {
|
|
|
383
373
|
// re-runs a finished cell — that is what prevents a second post). Membership
|
|
384
374
|
// comes from the materialized snapshot, so a config reload cannot add cells.
|
|
385
375
|
const pending = slotCtx ? coordinator.pendingTargets(slotCtx.slotId, targets) : targets.map((target) => ({ target, cell: null }));
|
|
386
|
-
|
|
376
|
+
const selected = onlyTarget ? pending.filter((p) => p.target.id === onlyTarget) : pending;
|
|
377
|
+
// Hand each cell its durable identity. Without this the handler cannot tell a
|
|
378
|
+
// first selection from a recovery, and would re-rank on resume — silently
|
|
379
|
+
// re-pointing the logical item at a different work than the one it owns.
|
|
380
|
+
const targetExecutionContexts = slotCtx
|
|
381
|
+
? coordinator.executionContextsFor(slotCtx.slotId, selected)
|
|
382
|
+
: undefined;
|
|
383
|
+
let runTargets = selected.map((p) => p.target);
|
|
387
384
|
if (slotCtx && runTargets.length === 0) {
|
|
388
385
|
logger_1.logger.info('All slot cells already complete', { slot: slotCtx.slotId });
|
|
389
386
|
coordinator.finish(slotCtx, schedule, targets);
|
|
@@ -424,6 +421,8 @@ async function createSchedulerRuntime(configPathArg) {
|
|
|
424
421
|
});
|
|
425
422
|
}
|
|
426
423
|
const downloadManager = new DownloadManager_1.DownloadManager(scopedConfig, pixivClient, database, fileService);
|
|
424
|
+
if (targetExecutionContexts)
|
|
425
|
+
downloadManager.setTargetExecutionContexts(targetExecutionContexts);
|
|
427
426
|
if (options.excludedWorkIds)
|
|
428
427
|
downloadManager.setProcessedWorkIds(options.excludedWorkIds);
|
|
429
428
|
activeDownloadManager = downloadManager;
|
|
@@ -35,6 +35,20 @@ export type DeliveryAck = {
|
|
|
35
35
|
matchedKey?: string;
|
|
36
36
|
raw?: unknown;
|
|
37
37
|
}
|
|
38
|
+
/**
|
|
39
|
+
* The provider accepted the request and persisted a record, but that record
|
|
40
|
+
* is in a TERMINAL FAILURE state (`failed`/`rejected`/`invalid`/`expired`).
|
|
41
|
+
* Nothing will ever be published from this intent, and because the provider
|
|
42
|
+
* keys its record by OUR idempotency_key, another attempt only returns the
|
|
43
|
+
* same failed record. Terminal: never retried, never reported as delivered.
|
|
44
|
+
*/
|
|
45
|
+
| {
|
|
46
|
+
kind: 'remote_failed';
|
|
47
|
+
remoteId?: string;
|
|
48
|
+
remoteStatus: string;
|
|
49
|
+
error: string;
|
|
50
|
+
raw?: unknown;
|
|
51
|
+
}
|
|
38
52
|
/** Transient failure: timeouts, 5xx, 429, connection errors. Retryable. */
|
|
39
53
|
| {
|
|
40
54
|
kind: 'retryable_failure';
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.parseDeliveryAck = parseDeliveryAck;
|
|
4
|
+
/** Downstream record states that will never become a published delivery. */
|
|
5
|
+
const TERMINAL_REMOTE_STATUSES = new Set(['failed', 'rejected', 'invalid', 'expired']);
|
|
4
6
|
const DEFAULT_HINTS = {
|
|
5
7
|
dataPath: 'data',
|
|
6
8
|
idField: 'review_id',
|
|
@@ -49,6 +51,22 @@ function parseDeliveryAck(status, body, hints = {}) {
|
|
|
49
51
|
const remoteId = typeof idValue === 'number' ? String(idValue) : asString(idValue);
|
|
50
52
|
const remoteStatus = asString(rec[h.statusField]);
|
|
51
53
|
const reused = rec[h.reusedField] === true;
|
|
54
|
+
// A 2xx envelope can still describe a RECORD THAT FAILED downstream: the
|
|
55
|
+
// provider persists the record before doing the real work, so "accepted" and
|
|
56
|
+
// "the persisted record is broken" arrive on the same HTTP status. The record
|
|
57
|
+
// is what gets published (and what a retry will find again), so its terminal
|
|
58
|
+
// state outranks the transport-level success of this request — otherwise a
|
|
59
|
+
// remote failure is recorded here as an end-to-end success.
|
|
60
|
+
const terminalStatus = remoteStatus?.trim().toLowerCase();
|
|
61
|
+
if (terminalStatus && TERMINAL_REMOTE_STATUSES.has(terminalStatus)) {
|
|
62
|
+
return {
|
|
63
|
+
kind: 'remote_failed',
|
|
64
|
+
remoteId,
|
|
65
|
+
remoteStatus: terminalStatus,
|
|
66
|
+
error: `delivery endpoint reported terminal status ${terminalStatus}`,
|
|
67
|
+
raw: body,
|
|
68
|
+
};
|
|
69
|
+
}
|
|
52
70
|
if (!reused) {
|
|
53
71
|
return { kind: 'accepted', remoteId, remoteStatus, raw: body };
|
|
54
72
|
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { Database } from '../storage/Database';
|
|
2
|
+
import { SchedulerDeliveryPort } from '../scheduler/SlotCoordinator';
|
|
3
|
+
/**
|
|
4
|
+
* Answers the slot FSM's only delivery question — "does this cell still have an
|
|
5
|
+
* owner downstream?" — from the durable delivery ledger.
|
|
6
|
+
*
|
|
7
|
+
* The distinction matters after a crash: a `delivery_pending` cell whose intent
|
|
8
|
+
* is still live must be left alone (the OutboxWorker retries THAT work to a
|
|
9
|
+
* terminal ACK), while one whose intent is terminally failed must be failed
|
|
10
|
+
* explicitly. Both are read from committed rows, so the answer is the same
|
|
11
|
+
* before and after a restart — which is exactly what a recovery decision needs.
|
|
12
|
+
*/
|
|
13
|
+
export declare function createDeliveryLedgerPort(database: Database): SchedulerDeliveryPort;
|
|
14
|
+
//# sourceMappingURL=DeliveryLedgerPort.d.ts.map
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.createDeliveryLedgerPort = createDeliveryLedgerPort;
|
|
4
|
+
/**
|
|
5
|
+
* Answers the slot FSM's only delivery question — "does this cell still have an
|
|
6
|
+
* owner downstream?" — from the durable delivery ledger.
|
|
7
|
+
*
|
|
8
|
+
* The distinction matters after a crash: a `delivery_pending` cell whose intent
|
|
9
|
+
* is still live must be left alone (the OutboxWorker retries THAT work to a
|
|
10
|
+
* terminal ACK), while one whose intent is terminally failed must be failed
|
|
11
|
+
* explicitly. Both are read from committed rows, so the answer is the same
|
|
12
|
+
* before and after a restart — which is exactly what a recovery decision needs.
|
|
13
|
+
*/
|
|
14
|
+
function createDeliveryLedgerPort(database) {
|
|
15
|
+
return {
|
|
16
|
+
stateFor({ deliveryTarget, slotId, targetId }) {
|
|
17
|
+
const rows = database.deliveries.listForCell(deliveryTarget, slotId, targetId);
|
|
18
|
+
if (rows.length === 0)
|
|
19
|
+
return { kind: 'unknown' };
|
|
20
|
+
const confirmed = rows.find((r) => r.status === 'delivered' || r.status === 'duplicate');
|
|
21
|
+
if (confirmed) {
|
|
22
|
+
return { kind: 'confirmed', workId: confirmed.pixivId, workType: confirmed.workType };
|
|
23
|
+
}
|
|
24
|
+
const pending = rows.filter((r) => r.status === 'pending');
|
|
25
|
+
// Still actionable = the outbox is going to try again. Its retry budget is
|
|
26
|
+
// the only thing that may converge this delivery, so keep hands off.
|
|
27
|
+
if (pending.some((r) => database.outbox.hasActionableDelivery(r.id))) {
|
|
28
|
+
return { kind: 'live' };
|
|
29
|
+
}
|
|
30
|
+
// Either the outbox exhausted its retries (dead/failed) or the intent was
|
|
31
|
+
// never queued. Nobody will converge it, and re-running selection would
|
|
32
|
+
// re-point the logical item at a different work — so fail it as-is.
|
|
33
|
+
if (pending.length > 0 || rows.some((r) => r.status === 'failed')) {
|
|
34
|
+
return {
|
|
35
|
+
kind: 'lost',
|
|
36
|
+
reason: 'delivery intent reached a terminal failure without an ACK; failing the cell instead of re-selecting a different work',
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
return { kind: 'unknown' };
|
|
40
|
+
},
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
//# sourceMappingURL=DeliveryLedgerPort.js.map
|
|
@@ -312,6 +312,19 @@ class OutboxWorker {
|
|
|
312
312
|
});
|
|
313
313
|
this.onDeliveryTerminal?.(row.deliveryId, ack, payload);
|
|
314
314
|
break;
|
|
315
|
+
case 'remote_failed':
|
|
316
|
+
// The provider persisted the record, but it will never publish and the
|
|
317
|
+
// idempotency key pins us to that same record forever: retrying cannot
|
|
318
|
+
// change the outcome. Terminal, so the failure reaches the Slot instead
|
|
319
|
+
// of being masked as a delivered cell. Content is kept for inspection.
|
|
320
|
+
this.database.deliveries.recordAck(row.deliveryId, {
|
|
321
|
+
status: 'failed',
|
|
322
|
+
remoteId: ack.remoteId,
|
|
323
|
+
remoteStatus: ack.remoteStatus,
|
|
324
|
+
error: ack.error,
|
|
325
|
+
});
|
|
326
|
+
this.onDeliveryTerminal?.(row.deliveryId, ack, payload);
|
|
327
|
+
break;
|
|
315
328
|
case 'permanent_failure':
|
|
316
329
|
// Deterministic rejection: still retry a couple times to survive a
|
|
317
330
|
// misconfigured blip, the outbox max-attempts then dead-letters it.
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Terminal-ACK settlement: the ONE place a confirmed downstream delivery ACK
|
|
3
|
+
* becomes a Slot cell state.
|
|
4
|
+
*
|
|
5
|
+
* The outbox worker already recorded the durable delivery ledger outcome before
|
|
6
|
+
* calling here; this maps that ack onto the Slot FSM. Keeping it as a named,
|
|
7
|
+
* dependency-light function (instead of an inline closure in the scheduler
|
|
8
|
+
* runtime) is what makes the central invariant directly testable:
|
|
9
|
+
*
|
|
10
|
+
* REMOTE FAILURE MUST NOT BE REPORTED AS END-TO-END SUCCESS.
|
|
11
|
+
*
|
|
12
|
+
* A provider can answer HTTP 2xx while the record it persisted is terminally
|
|
13
|
+
* broken. The provider keys that record by OUR idempotency key, so a retry only
|
|
14
|
+
* returns the same broken record. Such an ack must settle the cell as `failed`,
|
|
15
|
+
* never promote it to `submitted` as though the content had been published.
|
|
16
|
+
*/
|
|
17
|
+
import type { Database } from '../storage/Database';
|
|
18
|
+
import type { DeliveryAck } from './DeliveryAck';
|
|
19
|
+
/**
|
|
20
|
+
* Apply a terminal delivery ack to the Slot cell that owns the delivery intent.
|
|
21
|
+
* Returns false when the delivery has no Slot cell (ad-hoc / batch runs), which
|
|
22
|
+
* is a normal no-op rather than an error.
|
|
23
|
+
*/
|
|
24
|
+
export declare function settleDeliveryTerminal(database: Database, deliveryId: string, ack: DeliveryAck): boolean;
|
|
25
|
+
//# sourceMappingURL=settleDeliveryTerminal.d.ts.map
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.settleDeliveryTerminal = settleDeliveryTerminal;
|
|
4
|
+
const SlotCoordinator_1 = require("../scheduler/SlotCoordinator");
|
|
5
|
+
/**
|
|
6
|
+
* Apply a terminal delivery ack to the Slot cell that owns the delivery intent.
|
|
7
|
+
* Returns false when the delivery has no Slot cell (ad-hoc / batch runs), which
|
|
8
|
+
* is a normal no-op rather than an error.
|
|
9
|
+
*/
|
|
10
|
+
function settleDeliveryTerminal(database, deliveryId, ack) {
|
|
11
|
+
const row = database.deliveries.getById(deliveryId);
|
|
12
|
+
if (!row || !row.slotId || !row.targetId)
|
|
13
|
+
return false;
|
|
14
|
+
const coord = new SlotCoordinator_1.SlotCoordinator(database);
|
|
15
|
+
if (ack.kind === 'duplicate_existing') {
|
|
16
|
+
// Historical duplicate: a record from a DIFFERENT intent already exists, so
|
|
17
|
+
// THIS intent published nothing. Never a successful submission.
|
|
18
|
+
coord.applyOutcome(row.slotId, row.targetId, {
|
|
19
|
+
kind: 'duplicate',
|
|
20
|
+
workId: row.pixivId,
|
|
21
|
+
reason: 'downstream attested historical duplicate',
|
|
22
|
+
});
|
|
23
|
+
return true;
|
|
24
|
+
}
|
|
25
|
+
if (ack.kind === 'remote_failed') {
|
|
26
|
+
// The downstream record is terminally broken, so nothing was published:
|
|
27
|
+
// settle the cell as failed instead of promoting it to submitted. Not
|
|
28
|
+
// retryable — the same idempotency key can only return that record.
|
|
29
|
+
coord.applyOutcome(row.slotId, row.targetId, {
|
|
30
|
+
kind: 'failed',
|
|
31
|
+
retryable: false,
|
|
32
|
+
error: `downstream reported terminal status ${ack.remoteStatus} (review ${ack.remoteId ?? 'unknown'})`,
|
|
33
|
+
});
|
|
34
|
+
return true;
|
|
35
|
+
}
|
|
36
|
+
// accepted / idempotent_replay: a confirmed ACK is the sole path to submitted.
|
|
37
|
+
coord.markDelivered(row.slotId, row.targetId, row.pixivId, row.workType);
|
|
38
|
+
return true;
|
|
39
|
+
}
|
|
40
|
+
//# sourceMappingURL=settleDeliveryTerminal.js.map
|
|
@@ -5,6 +5,7 @@ import { IDatabase } from '../interfaces/IDatabase';
|
|
|
5
5
|
import { IFileService } from '../interfaces/IFileService';
|
|
6
6
|
import { DeliveryService } from '../delivery/DeliveryService';
|
|
7
7
|
import { TargetOutcome } from '../scheduler/TargetOutcome';
|
|
8
|
+
import { TargetExecutionContext } from '../scheduler/WorkIdentity';
|
|
8
9
|
/**
|
|
9
10
|
* Download Manager with Concurrency Control
|
|
10
11
|
*
|
|
@@ -76,6 +77,10 @@ export declare class DownloadManager implements IDownloadManager {
|
|
|
76
77
|
slotName: string;
|
|
77
78
|
slotDate: string;
|
|
78
79
|
};
|
|
80
|
+
/** Per-cell work identity for THIS run, keyed by target id (scheduled runs only). */
|
|
81
|
+
private targetExecutionContexts;
|
|
82
|
+
/** Publish the durable cell identity each handler must honour (scheduled runs). */
|
|
83
|
+
setTargetExecutionContexts(contexts: Map<string, TargetExecutionContext>): void;
|
|
79
84
|
/**
|
|
80
85
|
* Request cooperative cancellation of the current run. In-flight item
|
|
81
86
|
* finishes; no further targets/items are started. runAllTargets() will
|
|
@@ -86,6 +86,12 @@ class DownloadManager {
|
|
|
86
86
|
}
|
|
87
87
|
/** Slot context propagated to delivery templates/outcomes for scheduled runs. */
|
|
88
88
|
slotContext;
|
|
89
|
+
/** Per-cell work identity for THIS run, keyed by target id (scheduled runs only). */
|
|
90
|
+
targetExecutionContexts = new Map();
|
|
91
|
+
/** Publish the durable cell identity each handler must honour (scheduled runs). */
|
|
92
|
+
setTargetExecutionContexts(contexts) {
|
|
93
|
+
this.targetExecutionContexts = contexts;
|
|
94
|
+
}
|
|
89
95
|
/**
|
|
90
96
|
* Request cooperative cancellation of the current run. In-flight item
|
|
91
97
|
* finishes; no further targets/items are started. runAllTargets() will
|
|
@@ -216,11 +222,15 @@ class DownloadManager {
|
|
|
216
222
|
}
|
|
217
223
|
}
|
|
218
224
|
async dispatchTarget(target) {
|
|
225
|
+
// The cell identity must reach the handler or it cannot tell a first
|
|
226
|
+
// selection from a recovery: it would re-rank and silently re-point the
|
|
227
|
+
// logical item at a different work.
|
|
228
|
+
const execution = target.id ? this.targetExecutionContexts.get(target.id) : undefined;
|
|
219
229
|
switch (target.type) {
|
|
220
230
|
case 'illustration':
|
|
221
|
-
return await this.illustrationHandler.handle(target);
|
|
231
|
+
return await this.illustrationHandler.handle(target, execution);
|
|
222
232
|
case 'novel':
|
|
223
|
-
return await this.novelHandler.handle(target);
|
|
233
|
+
return await this.novelHandler.handle(target, execution);
|
|
224
234
|
default:
|
|
225
235
|
logger_1.logger.warn(`Unsupported target type ${target.type}`);
|
|
226
236
|
return { kind: 'failed', retryable: false, error: `unsupported target type ${target.type}` };
|
|
@@ -6,6 +6,7 @@ import { IllustrationDownloader } from '../IllustrationDownloader';
|
|
|
6
6
|
import { DownloadPipeline } from '../pipeline/DownloadPipeline';
|
|
7
7
|
import { DeliveryService } from '../../delivery/DeliveryService';
|
|
8
8
|
import { TargetOutcome } from '../../scheduler/TargetOutcome';
|
|
9
|
+
import { TargetExecutionContext } from '../../scheduler/WorkIdentity';
|
|
9
10
|
import type { TopicPipelineFactory } from '../../topic/createTopicPipeline';
|
|
10
11
|
export declare class IllustrationTargetHandler {
|
|
11
12
|
private readonly client;
|
|
@@ -17,8 +18,14 @@ export declare class IllustrationTargetHandler {
|
|
|
17
18
|
private readonly deliveryService?;
|
|
18
19
|
/** Outcomes produced during this handle() call (deliveries + terminal non-matches). */
|
|
19
20
|
private outcomes;
|
|
21
|
+
/**
|
|
22
|
+
* Cell identity for this handle() call. Set only for a single-work cell of a
|
|
23
|
+
* scheduled occurrence; null for ad-hoc runs and N-works-per-run targets, which
|
|
24
|
+
* have no single (slotId,targetId) -> workId identity to honour.
|
|
25
|
+
*/
|
|
26
|
+
private execution;
|
|
20
27
|
constructor(client: IPixivClient, database: IDatabase, rankingService: RankingService, illustrationDownloader: IllustrationDownloader, pipeline: DownloadPipeline, topicPipelineFactory?: TopicPipelineFactory | undefined, deliveryService?: DeliveryService | undefined);
|
|
21
|
-
handle(target: TargetConfig): Promise<TargetOutcome>;
|
|
28
|
+
handle(target: TargetConfig, execution?: TargetExecutionContext): Promise<TargetOutcome>;
|
|
22
29
|
/** Reduce the outcomes collected while processing one target to one cell result. */
|
|
23
30
|
private summarize;
|
|
24
31
|
private classifyError;
|
|
@@ -35,6 +42,25 @@ export declare class IllustrationTargetHandler {
|
|
|
35
42
|
private handleUserIllustrations;
|
|
36
43
|
private sortByPopularityAndLog;
|
|
37
44
|
private logError;
|
|
45
|
+
/**
|
|
46
|
+
* Continue the work this cell ALREADY owns. Selection is deliberately skipped:
|
|
47
|
+
* no search, no ranking, no topic expansion, no backfill pool, and no
|
|
48
|
+
* "already downloaded / already delivered" exclusion — the cell's own work must
|
|
49
|
+
* never be filtered out of its own recovery.
|
|
50
|
+
*
|
|
51
|
+
* A locked work that is permanently gone is a terminal failure of THIS logical
|
|
52
|
+
* item (`LOCKED_WORK_UNAVAILABLE`). It is never silently replaced by another
|
|
53
|
+
* candidate; that would mutate the item's identity behind the operator's back.
|
|
54
|
+
*/
|
|
55
|
+
private recoverLockedWork;
|
|
56
|
+
/**
|
|
57
|
+
* Process ONE candidate work for this cell.
|
|
58
|
+
*
|
|
59
|
+
* The cell is bound to `illust` BEFORE any side effect: it is the binding, not
|
|
60
|
+
* the candidate list, that decides what a later recovery resumes. The binding
|
|
61
|
+
* is only rolled back when the attempt produced no artifact at all, so in-run
|
|
62
|
+
* backfill still works for a cell that has committed to nothing.
|
|
63
|
+
*/
|
|
38
64
|
private downloadAndDeliver;
|
|
39
65
|
/**
|
|
40
66
|
* Turn a downloaded artifact into the target's business outcome. In cache
|
|
@@ -5,6 +5,7 @@ const logger_1 = require("../../logger");
|
|
|
5
5
|
const pixiv_date_utils_1 = require("../../utils/pixiv-date-utils");
|
|
6
6
|
const errors_1 = require("../../utils/errors");
|
|
7
7
|
const pixiv_utils_1 = require("../../utils/pixiv-utils");
|
|
8
|
+
const WorkIdentity_1 = require("../../scheduler/WorkIdentity");
|
|
8
9
|
const target_label_1 = require("../../utils/target-label");
|
|
9
10
|
class IllustrationTargetHandler {
|
|
10
11
|
client;
|
|
@@ -16,6 +17,12 @@ class IllustrationTargetHandler {
|
|
|
16
17
|
deliveryService;
|
|
17
18
|
/** Outcomes produced during this handle() call (deliveries + terminal non-matches). */
|
|
18
19
|
outcomes = [];
|
|
20
|
+
/**
|
|
21
|
+
* Cell identity for this handle() call. Set only for a single-work cell of a
|
|
22
|
+
* scheduled occurrence; null for ad-hoc runs and N-works-per-run targets, which
|
|
23
|
+
* have no single (slotId,targetId) -> workId identity to honour.
|
|
24
|
+
*/
|
|
25
|
+
execution = null;
|
|
19
26
|
constructor(client, database, rankingService, illustrationDownloader, pipeline, topicPipelineFactory, deliveryService) {
|
|
20
27
|
this.client = client;
|
|
21
28
|
this.database = database;
|
|
@@ -25,8 +32,17 @@ class IllustrationTargetHandler {
|
|
|
25
32
|
this.topicPipelineFactory = topicPipelineFactory;
|
|
26
33
|
this.deliveryService = deliveryService;
|
|
27
34
|
}
|
|
28
|
-
async handle(target) {
|
|
35
|
+
async handle(target, execution) {
|
|
29
36
|
this.outcomes = [];
|
|
37
|
+
this.execution = execution && (0, WorkIdentity_1.isSingleWorkCell)(target) ? execution : null;
|
|
38
|
+
// A cell that already owns a work is in RECOVERY, not in a new selection.
|
|
39
|
+
// Crash/shutdown recovery is not an intentional second run: running the
|
|
40
|
+
// candidate pipeline here would re-rank and could bind this logical item to a
|
|
41
|
+
// different work than the one it already committed to.
|
|
42
|
+
if (this.execution?.lockedWorkId) {
|
|
43
|
+
await this.recoverLockedWork(target, this.execution.lockedWorkId);
|
|
44
|
+
return this.summarize(target);
|
|
45
|
+
}
|
|
30
46
|
if (target.illustId) {
|
|
31
47
|
await this.handleSingleIllustration(target);
|
|
32
48
|
return this.summarize(target);
|
|
@@ -361,14 +377,96 @@ class IllustrationTargetHandler {
|
|
|
361
377
|
stack: error instanceof Error ? error.stack : undefined,
|
|
362
378
|
});
|
|
363
379
|
}
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
380
|
+
/**
|
|
381
|
+
* Continue the work this cell ALREADY owns. Selection is deliberately skipped:
|
|
382
|
+
* no search, no ranking, no topic expansion, no backfill pool, and no
|
|
383
|
+
* "already downloaded / already delivered" exclusion — the cell's own work must
|
|
384
|
+
* never be filtered out of its own recovery.
|
|
385
|
+
*
|
|
386
|
+
* A locked work that is permanently gone is a terminal failure of THIS logical
|
|
387
|
+
* item (`LOCKED_WORK_UNAVAILABLE`). It is never silently replaced by another
|
|
388
|
+
* candidate; that would mutate the item's identity behind the operator's back.
|
|
389
|
+
*/
|
|
390
|
+
async recoverLockedWork(target, lockedWorkId) {
|
|
391
|
+
const displayTag = (0, target_label_1.getTargetLabel)(target);
|
|
392
|
+
logger_1.logger.info(`Recovering locked work ${lockedWorkId} for ${displayTag}; candidate selection is skipped`, {
|
|
393
|
+
slotId: this.execution?.slotId,
|
|
394
|
+
targetId: this.execution?.targetId,
|
|
395
|
+
lockedWorkId,
|
|
369
396
|
});
|
|
370
|
-
|
|
397
|
+
const illustId = Number(lockedWorkId);
|
|
398
|
+
if (!Number.isFinite(illustId)) {
|
|
399
|
+
this.outcomes.push({
|
|
400
|
+
kind: 'failed',
|
|
401
|
+
retryable: false,
|
|
402
|
+
error: `LOCKED_WORK_UNAVAILABLE: cell work id "${lockedWorkId}" is not a valid illustration id`,
|
|
403
|
+
});
|
|
371
404
|
return;
|
|
405
|
+
}
|
|
406
|
+
try {
|
|
407
|
+
const detail = await this.client.getIllustration(illustId);
|
|
408
|
+
await this.downloadAndDeliver(detail, displayTag, target);
|
|
409
|
+
}
|
|
410
|
+
catch (error) {
|
|
411
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
412
|
+
const retryable = (0, errors_1.isRetryableNetworkError)(error);
|
|
413
|
+
this.logError(error, `Failed to recover locked illustration ${lockedWorkId}`);
|
|
414
|
+
this.outcomes.push({
|
|
415
|
+
kind: 'failed',
|
|
416
|
+
retryable,
|
|
417
|
+
error: retryable
|
|
418
|
+
? `locked work ${lockedWorkId} could not be fetched (will retry the SAME work): ${message}`
|
|
419
|
+
: `LOCKED_WORK_UNAVAILABLE: locked work ${lockedWorkId} is no longer fetchable: ${message}`,
|
|
420
|
+
});
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
/**
|
|
424
|
+
* Process ONE candidate work for this cell.
|
|
425
|
+
*
|
|
426
|
+
* The cell is bound to `illust` BEFORE any side effect: it is the binding, not
|
|
427
|
+
* the candidate list, that decides what a later recovery resumes. The binding
|
|
428
|
+
* is only rolled back when the attempt produced no artifact at all, so in-run
|
|
429
|
+
* backfill still works for a cell that has committed to nothing.
|
|
430
|
+
*/
|
|
431
|
+
async downloadAndDeliver(illust, tag, target) {
|
|
432
|
+
const execution = this.execution;
|
|
433
|
+
const workId = String(illust.id);
|
|
434
|
+
// Recovery continues a work the cell already bound; it must never be released.
|
|
435
|
+
const recovering = Boolean(execution?.lockedWorkId);
|
|
436
|
+
if (execution && !recovering) {
|
|
437
|
+
const binding = execution.bind(workId, 'illustration');
|
|
438
|
+
if (!binding.won) {
|
|
439
|
+
// Another writer elected a different work for this cell. First selection
|
|
440
|
+
// is authoritative: never process a work the cell does not own.
|
|
441
|
+
logger_1.logger.warn(`Cell is bound to work ${binding.workId}; declining to select ${workId}`, {
|
|
442
|
+
slotId: execution.slotId,
|
|
443
|
+
targetId: execution.targetId,
|
|
444
|
+
boundWorkId: binding.workId,
|
|
445
|
+
});
|
|
446
|
+
return;
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
let artifact = null;
|
|
450
|
+
try {
|
|
451
|
+
artifact = await this.illustrationDownloader.downloadIllustration(illust, tag, {
|
|
452
|
+
aiMetadataCheck: target.aiMetadataCheck === true,
|
|
453
|
+
maxPageCount: target.maxPageCount,
|
|
454
|
+
includeDeliveryPreviews: Boolean(target.storageMode === 'cache' && target.delivery?.target?.trim()),
|
|
455
|
+
});
|
|
456
|
+
}
|
|
457
|
+
catch (error) {
|
|
458
|
+
// Nothing was persisted, so the cell may still pick another candidate.
|
|
459
|
+
if (execution && !recovering)
|
|
460
|
+
execution.release(workId);
|
|
461
|
+
throw error;
|
|
462
|
+
}
|
|
463
|
+
if (!artifact) {
|
|
464
|
+
if (execution && !recovering)
|
|
465
|
+
execution.release(workId);
|
|
466
|
+
return;
|
|
467
|
+
}
|
|
468
|
+
// Committed: the artifact is durable and this work now defines the cell. Even
|
|
469
|
+
// if enqueue throws below, recovery must resume THIS work — never release.
|
|
372
470
|
this.recordArtifactOutcome(artifact, target);
|
|
373
471
|
}
|
|
374
472
|
/**
|
|
@@ -6,6 +6,7 @@ import { DownloadPipeline } from '../pipeline/DownloadPipeline';
|
|
|
6
6
|
import { NovelDownloader } from '../NovelDownloader';
|
|
7
7
|
import { DeliveryService } from '../../delivery/DeliveryService';
|
|
8
8
|
import { TargetOutcome } from '../../scheduler/TargetOutcome';
|
|
9
|
+
import { TargetExecutionContext } from '../../scheduler/WorkIdentity';
|
|
9
10
|
import type { TopicPipelineFactory } from '../../topic/createTopicPipeline';
|
|
10
11
|
export declare class NovelTargetHandler {
|
|
11
12
|
private readonly client;
|
|
@@ -16,8 +17,14 @@ export declare class NovelTargetHandler {
|
|
|
16
17
|
private readonly topicPipelineFactory?;
|
|
17
18
|
private readonly deliveryService?;
|
|
18
19
|
private outcomes;
|
|
20
|
+
/**
|
|
21
|
+
* Cell identity for this handle() call. Set only for a single-work cell of a
|
|
22
|
+
* scheduled occurrence; null for ad-hoc runs and N-works-per-run targets, which
|
|
23
|
+
* have no single (slotId,targetId) -> workId identity to honour.
|
|
24
|
+
*/
|
|
25
|
+
private execution;
|
|
19
26
|
constructor(client: IPixivClient, database: IDatabase, rankingService: RankingService, pipeline: DownloadPipeline, novelDownloader: NovelDownloader, topicPipelineFactory?: TopicPipelineFactory | undefined, deliveryService?: DeliveryService | undefined);
|
|
20
|
-
handle(target: TargetConfig): Promise<TargetOutcome>;
|
|
27
|
+
handle(target: TargetConfig, execution?: TargetExecutionContext): Promise<TargetOutcome>;
|
|
21
28
|
private summarize;
|
|
22
29
|
private classifyError;
|
|
23
30
|
private fetchNovels;
|
|
@@ -34,6 +41,25 @@ export declare class NovelTargetHandler {
|
|
|
34
41
|
private handleUserNovels;
|
|
35
42
|
private sortByPopularityAndLog;
|
|
36
43
|
private logError;
|
|
44
|
+
/**
|
|
45
|
+
* Continue the work this cell ALREADY owns. Selection is deliberately skipped:
|
|
46
|
+
* no search, no ranking, no topic expansion, no backfill pool, and no
|
|
47
|
+
* "already downloaded / already delivered" exclusion — the cell's own work must
|
|
48
|
+
* never be filtered out of its own recovery.
|
|
49
|
+
*
|
|
50
|
+
* A locked work that is permanently gone is a terminal failure of THIS logical
|
|
51
|
+
* item (`LOCKED_WORK_UNAVAILABLE`). It is never silently replaced by another
|
|
52
|
+
* candidate; that would mutate the item's identity behind the operator's back.
|
|
53
|
+
*/
|
|
54
|
+
private recoverLockedWork;
|
|
55
|
+
/**
|
|
56
|
+
* Process ONE candidate work for this cell.
|
|
57
|
+
*
|
|
58
|
+
* The cell is bound to `novel` BEFORE any side effect: it is the binding, not
|
|
59
|
+
* the candidate list, that decides what a later recovery resumes. The binding
|
|
60
|
+
* is only rolled back when the attempt produced no artifact at all, so in-run
|
|
61
|
+
* backfill still works for a cell that has committed to nothing.
|
|
62
|
+
*/
|
|
37
63
|
private downloadAndDeliver;
|
|
38
64
|
}
|
|
39
65
|
//# sourceMappingURL=NovelTargetHandler.d.ts.map
|