pixivflow 2.19.4 → 2.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/SchedulerCommand.js +36 -0
- package/dist/commands/SchedulerRunOnceCommand.d.ts +2 -3
- package/dist/commands/SchedulerRunOnceCommand.js +2 -3
- package/dist/commands/scheduler-runtime.js +17 -1
- package/dist/config/defaults.d.ts +1 -0
- package/dist/config/defaults.js +4 -0
- package/dist/config/environment.js +19 -0
- package/dist/config/types.d.ts +28 -0
- package/dist/config/validation.js +25 -0
- package/dist/delivery/DeliveryService.d.ts +30 -1
- package/dist/delivery/DeliveryService.js +13 -2
- package/dist/delivery/HttpMultipartDelivery.js +29 -8
- package/dist/delivery/OutboxWorker.d.ts +2 -0
- package/dist/delivery/OutboxWorker.js +3 -0
- package/dist/delivery/types.d.ts +18 -0
- package/dist/download/DownloadManager.js +12 -1
- package/dist/download/handlers/IllustrationTargetHandler.d.ts +42 -2
- package/dist/download/handlers/IllustrationTargetHandler.js +240 -50
- package/dist/download/handlers/NovelTargetHandler.d.ts +47 -0
- package/dist/download/handlers/NovelTargetHandler.js +213 -27
- package/dist/download/handlers/deliveryContext.d.ts +16 -0
- package/dist/download/handlers/deliveryContext.js +32 -0
- package/dist/download/pipeline/DownloadPipeline.d.ts +22 -3
- package/dist/download/pipeline/DownloadPipeline.js +109 -92
- package/dist/download/plan/DownloadPlanner.d.ts +47 -1
- package/dist/download/plan/DownloadPlanner.js +105 -15
- package/dist/notification/NotificationPolicy.d.ts +12 -0
- package/dist/notification/NotificationPolicy.js +86 -0
- package/dist/package.json +1 -1
- package/dist/scheduler/MultiScheduleManager.js +2 -0
- package/dist/scheduler/ScheduleTriggerServer.d.ts +58 -1
- package/dist/scheduler/ScheduleTriggerServer.js +207 -40
- package/dist/scheduler/SlotCoordinator.d.ts +80 -2
- package/dist/scheduler/SlotCoordinator.js +145 -3
- package/dist/scheduler/TargetOutcome.d.ts +195 -2
- package/dist/scheduler/TargetOutcome.js +202 -3
- package/dist/storage/DatabaseMigration.js +11 -0
- package/dist/storage/repositories/DeliveryRepository.d.ts +12 -0
- package/dist/storage/repositories/DeliveryRepository.js +18 -1
- package/dist/storage/repositories/SlotRepository.d.ts +8 -0
- package/dist/storage/repositories/SlotRepository.js +6 -2
- package/dist/version.js +1 -1
- package/dist/webui/package.json +1 -1
- package/package.json +1 -1
|
@@ -5,8 +5,11 @@ const logger_1 = require("../../logger");
|
|
|
5
5
|
const pixiv_date_utils_1 = require("../../utils/pixiv-date-utils");
|
|
6
6
|
const errors_1 = require("../../utils/errors");
|
|
7
7
|
const pixiv_utils_1 = require("../../utils/pixiv-utils");
|
|
8
|
+
const TargetOutcome_1 = require("../../scheduler/TargetOutcome");
|
|
8
9
|
const WorkIdentity_1 = require("../../scheduler/WorkIdentity");
|
|
9
10
|
const target_label_1 = require("../../utils/target-label");
|
|
11
|
+
const DownloadPlanner_1 = require("../plan/DownloadPlanner");
|
|
12
|
+
const deliveryContext_1 = require("./deliveryContext");
|
|
10
13
|
class IllustrationTargetHandler {
|
|
11
14
|
client;
|
|
12
15
|
database;
|
|
@@ -17,6 +20,34 @@ class IllustrationTargetHandler {
|
|
|
17
20
|
deliveryService;
|
|
18
21
|
/** Outcomes produced during this handle() call (deliveries + terminal non-matches). */
|
|
19
22
|
outcomes = [];
|
|
23
|
+
/**
|
|
24
|
+
* Candidate scan of the last pipeline.run() of this handle() call: how many
|
|
25
|
+
* candidates were attempted, which were skipped and why, and whether any
|
|
26
|
+
* job-level outage appeared. This is what turns an empty result into an
|
|
27
|
+
* explicit verdict instead of an ambiguous "completed".
|
|
28
|
+
*/
|
|
29
|
+
scan = null;
|
|
30
|
+
/**
|
|
31
|
+
* Global candidate-scan bound (`download.candidateScanLimit`), supplied by
|
|
32
|
+
* DownloadManager. This is what lets the FETCH stage ask for more than one
|
|
33
|
+
* candidate: a scheduled one-post-per-slot target has `limit: 1`, and asking
|
|
34
|
+
* the ranking API for exactly one work is the reason a single duplicate used
|
|
35
|
+
* to be unfixable downstream.
|
|
36
|
+
*/
|
|
37
|
+
defaultCandidateScanLimit;
|
|
38
|
+
/** Publish the global candidate-scan bound for the fetch stage. */
|
|
39
|
+
setDefaultCandidateScanLimit(limit) {
|
|
40
|
+
this.defaultCandidateScanLimit = limit;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* How many candidates to FETCH so the bounded scan has something to choose
|
|
44
|
+
* from. Same rule the planner applies to the attempt window, so fetch and
|
|
45
|
+
* scan agree on one bound.
|
|
46
|
+
*/
|
|
47
|
+
candidateFetchLimit(target) {
|
|
48
|
+
const targetLimit = target.limit && target.limit > 0 ? target.limit : 10;
|
|
49
|
+
return Math.max(targetLimit, (0, DownloadPlanner_1.resolveCandidateScanLimit)(target, this.defaultCandidateScanLimit));
|
|
50
|
+
}
|
|
20
51
|
/**
|
|
21
52
|
* Cell identity for this handle() call. Set only for a single-work cell of a
|
|
22
53
|
* scheduled occurrence; null for ad-hoc runs and N-works-per-run targets, which
|
|
@@ -34,6 +65,7 @@ class IllustrationTargetHandler {
|
|
|
34
65
|
}
|
|
35
66
|
async handle(target, execution) {
|
|
36
67
|
this.outcomes = [];
|
|
68
|
+
this.scan = null;
|
|
37
69
|
this.execution = execution && (0, WorkIdentity_1.isSingleWorkCell)(target) ? execution : null;
|
|
38
70
|
// A cell that already owns a work is in RECOVERY, not in a new selection.
|
|
39
71
|
// Crash/shutdown recovery is not an intentional second run: running the
|
|
@@ -61,6 +93,7 @@ class IllustrationTargetHandler {
|
|
|
61
93
|
}
|
|
62
94
|
const illusts = await this.fetchIllustrations(target, mode);
|
|
63
95
|
const result = await this.pipeline.run(illusts, target, 'illustration', (illust, tag) => this.downloadAndDeliver(illust, tag, target));
|
|
96
|
+
this.scan = result.scan;
|
|
64
97
|
this.handleDownloadResult(result, target, mode, illusts.length);
|
|
65
98
|
return this.summarize(target);
|
|
66
99
|
}
|
|
@@ -68,40 +101,89 @@ class IllustrationTargetHandler {
|
|
|
68
101
|
return this.classifyError(error, displayTag, mode, target);
|
|
69
102
|
}
|
|
70
103
|
}
|
|
71
|
-
/**
|
|
104
|
+
/**
|
|
105
|
+
* Reduce the outcomes collected while processing one target to one cell
|
|
106
|
+
* result, including the bounded-scan bookkeeping.
|
|
107
|
+
*
|
|
108
|
+
* A `duplicate` is deliberately NOT a target verdict here: a duplicate is a
|
|
109
|
+
* CANDIDATE problem (skip it and try the next candidate), which is what the
|
|
110
|
+
* scan already did. Returning it as the target outcome is the bug that made a
|
|
111
|
+
* scheduled slot report success after submitting nothing.
|
|
112
|
+
*/
|
|
72
113
|
summarize(target) {
|
|
114
|
+
const scan = this.scan ?? undefined;
|
|
73
115
|
const submitted = this.outcomes.find((o) => o.kind === 'submitted');
|
|
74
116
|
if (submitted)
|
|
75
|
-
return submitted;
|
|
117
|
+
return scan ? { ...submitted, scan } : submitted;
|
|
76
118
|
const stored = this.outcomes.find((o) => o.kind === 'stored');
|
|
77
119
|
if (stored)
|
|
78
|
-
return stored;
|
|
120
|
+
return scan ? { ...stored, scan } : stored;
|
|
79
121
|
const pending = this.outcomes.find((o) => o.kind === 'delivery_pending');
|
|
80
122
|
if (pending)
|
|
81
|
-
return pending;
|
|
82
|
-
const
|
|
83
|
-
if (
|
|
84
|
-
return
|
|
85
|
-
const
|
|
86
|
-
if (
|
|
87
|
-
return
|
|
88
|
-
|
|
123
|
+
return scan ? { ...pending, scan } : pending;
|
|
124
|
+
const alreadyFailed = this.outcomes.find((o) => o.kind === 'failed');
|
|
125
|
+
if (alreadyFailed)
|
|
126
|
+
return scan ? { ...alreadyFailed, scan } : alreadyFailed;
|
|
127
|
+
const terminalDuplicate = this.outcomes.find((o) => o.kind === 'duplicate');
|
|
128
|
+
if (terminalDuplicate)
|
|
129
|
+
return scan ? { ...terminalDuplicate, scan } : terminalDuplicate;
|
|
130
|
+
// A job-level outage is never an empty candidate list. Failing here keeps
|
|
131
|
+
// the existing retry/backoff semantics: the scheduler retries the job on a
|
|
132
|
+
// later trigger instead of recording a clean no-op.
|
|
133
|
+
if (scan && scan.outages.length > 0) {
|
|
134
|
+
return {
|
|
135
|
+
kind: 'failed',
|
|
136
|
+
retryable: true,
|
|
137
|
+
error: `job-level outage while scanning candidates: ${scan.outages.join(', ')}`,
|
|
138
|
+
scan,
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
if (scan && (0, TargetOutcome_1.hasTransientFailure)(scan)) {
|
|
142
|
+
return {
|
|
143
|
+
kind: 'failed',
|
|
144
|
+
retryable: true,
|
|
145
|
+
error: `candidate scan hit transient infrastructure failures ` +
|
|
146
|
+
`(${scan.skipped.filter((s) => s.retryable).length} of ${scan.attempted} attempted)`,
|
|
147
|
+
scan,
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
if (scan && (scan.attempted > 0 || scan.skipped.length > 0)) {
|
|
151
|
+
return { kind: 'no_candidate', reason: (0, TargetOutcome_1.noEligibleCandidateText)(scan), scan };
|
|
152
|
+
}
|
|
153
|
+
return {
|
|
154
|
+
kind: 'no_candidate',
|
|
155
|
+
reason: 'no matching illustration after filtering/dedupe',
|
|
156
|
+
...(scan ? { scan } : {}),
|
|
157
|
+
};
|
|
89
158
|
}
|
|
90
159
|
classifyError(error, displayTag, mode, target) {
|
|
91
160
|
const message = error instanceof Error ? error.message : String(error);
|
|
161
|
+
const scan = this.scan ?? undefined;
|
|
162
|
+
// A hard job-level outage is named as such and is never recorded as a
|
|
163
|
+
// no-candidate business outcome, whatever its message happens to look like.
|
|
164
|
+
const outage = (0, TargetOutcome_1.classifyJobLevelOutage)(error);
|
|
92
165
|
this.database.logExecution(displayTag, 'illustration', 'failed', message);
|
|
93
166
|
logger_1.logger.error(`Illustration ${mode === 'ranking' ? 'ranking' : 'tag'} ${displayTag} failed`, {
|
|
94
167
|
error: message,
|
|
95
168
|
errorType: error instanceof Error ? error.constructor.name : typeof error,
|
|
169
|
+
...(outage ? { jobLevelOutage: outage } : {}),
|
|
96
170
|
});
|
|
171
|
+
if (outage) {
|
|
172
|
+
return {
|
|
173
|
+
kind: 'failed',
|
|
174
|
+
retryable: true,
|
|
175
|
+
error: `job-level outage (${outage}): ${message}`,
|
|
176
|
+
...(scan ? { scan } : {}),
|
|
177
|
+
};
|
|
178
|
+
}
|
|
97
179
|
// Explicit no-candidate signals are business outcomes, not failures.
|
|
98
180
|
if (/no matching|all .*filtered|no_candidate/i.test(message)) {
|
|
99
|
-
return { kind: 'no_candidate', reason: message };
|
|
181
|
+
return { kind: 'no_candidate', reason: message, ...(scan ? { scan } : {}) };
|
|
100
182
|
}
|
|
101
183
|
// Network/transient => retryable so the SAME work resumes on next trigger.
|
|
102
184
|
const retryable = (0, errors_1.isRetryableNetworkError)(error) ||
|
|
103
185
|
(error instanceof Error && /timeout|econn|enotfound|etimed|429|5\d\d/i.test(error.message));
|
|
104
|
-
return { kind: 'failed', retryable, error: message };
|
|
186
|
+
return { kind: 'failed', retryable, error: message, ...(scan ? { scan } : {}) };
|
|
105
187
|
}
|
|
106
188
|
async fetchIllustrations(target, mode) {
|
|
107
189
|
if (mode === 'topic') {
|
|
@@ -150,15 +232,37 @@ class IllustrationTargetHandler {
|
|
|
150
232
|
}
|
|
151
233
|
const illusts = await this.fetchTopicIllustrations(attemptTarget);
|
|
152
234
|
const result = await this.pipeline.run(illusts, attemptTarget, 'illustration', (illust, tag) => this.downloadAndDeliver(illust, tag, attemptTarget));
|
|
235
|
+
// The lookback loop is one bounded scan: accumulate every day's
|
|
236
|
+
// skips/outages so the final verdict covers all candidates attempted.
|
|
237
|
+
this.scan = (0, TargetOutcome_1.mergeScanSummaries)(this.scan, result.scan);
|
|
153
238
|
if (result.downloaded > 0) {
|
|
154
239
|
this.handleDownloadResult(result, target, 'topic', illusts.length);
|
|
155
240
|
return;
|
|
156
241
|
}
|
|
242
|
+
if (this.scan && this.scan.outages.length > 0) {
|
|
243
|
+
// A dead token / dead database / dead network is not "no matching
|
|
244
|
+
// illustration": stop looking back and let the job fail/retry.
|
|
245
|
+
return;
|
|
246
|
+
}
|
|
157
247
|
}
|
|
158
|
-
const
|
|
248
|
+
const scan = this.scan;
|
|
249
|
+
// An explicit no-eligible-candidate verdict needs something to have been
|
|
250
|
+
// considered. When the scan is empty (nothing surfaced at all) the richer
|
|
251
|
+
// existing message — which names the days actually checked — is the honest
|
|
252
|
+
// one, so it is kept.
|
|
253
|
+
const considered = Boolean(scan && (scan.attempted > 0 || scan.skipped.length > 0));
|
|
254
|
+
const message = considered
|
|
255
|
+
? `${(0, TargetOutcome_1.noEligibleCandidateText)(scan)} (checked ${checkedDays.join(', ')})`
|
|
256
|
+
: `No matching illustrations found after checking ${checkedDays.length} day(s): ${checkedDays.join(', ')}`;
|
|
257
|
+
// A bounded, exhausted scan IS the success path for a no-op day: nothing
|
|
258
|
+
// was eligible and nothing was submitted. Recorded as an explicit verdict.
|
|
159
259
|
this.database.logExecution(displayTag, 'illustration', 'success', message);
|
|
160
|
-
logger_1.logger.warn(`Illustration topic ${displayTag} produced no matching result`, { checkedDays });
|
|
161
|
-
this.outcomes.push({
|
|
260
|
+
logger_1.logger.warn(`Illustration topic ${displayTag} produced no matching result`, { checkedDays, message });
|
|
261
|
+
this.outcomes.push({
|
|
262
|
+
kind: 'no_candidate',
|
|
263
|
+
reason: message,
|
|
264
|
+
...(scan ? { scan } : {}),
|
|
265
|
+
});
|
|
162
266
|
}
|
|
163
267
|
resolveTopicDay(target) {
|
|
164
268
|
return target.date === 'TODAY'
|
|
@@ -189,12 +293,14 @@ class IllustrationTargetHandler {
|
|
|
189
293
|
endDate: rankingDate,
|
|
190
294
|
limit: Math.max(targetLimit * 20, 100),
|
|
191
295
|
};
|
|
296
|
+
const fetchLimit = this.candidateFetchLimit(target);
|
|
192
297
|
let illusts = await this.client.searchIllustrations(searchTarget);
|
|
193
298
|
logger_1.logger.info(`Found ${illusts.length} illustration(s) for ${rankingDate}`);
|
|
194
|
-
this.sortByPopularityAndLog(illusts,
|
|
195
|
-
if (illusts.length >
|
|
196
|
-
illusts = illusts.slice(0,
|
|
197
|
-
logger_1.logger.info(`Selected top ${illusts.length} illustration(s) by popularity`
|
|
299
|
+
this.sortByPopularityAndLog(illusts, fetchLimit, 'illustration');
|
|
300
|
+
if (illusts.length > fetchLimit) {
|
|
301
|
+
illusts = illusts.slice(0, fetchLimit);
|
|
302
|
+
logger_1.logger.info(`Selected top ${illusts.length} illustration(s) by popularity ` +
|
|
303
|
+
`(to fill ${targetLimit}, candidate scan bound ${fetchLimit})`);
|
|
198
304
|
}
|
|
199
305
|
return illusts;
|
|
200
306
|
}
|
|
@@ -205,7 +311,9 @@ class IllustrationTargetHandler {
|
|
|
205
311
|
rankingDate = (0, pixiv_date_utils_1.getYesterdayDate)();
|
|
206
312
|
}
|
|
207
313
|
logger_1.logger.info(`Fetching ranking illustrations (mode: ${rankingMode}, date: ${rankingDate})`);
|
|
208
|
-
|
|
314
|
+
// Ask for the whole bounded scan window, not `limit`: the planner then
|
|
315
|
+
// narrows it to the candidates this run may attempt.
|
|
316
|
+
const illusts = await this.rankingService.getRankingIllustrationsWithFallback(rankingMode, rankingDate, this.candidateFetchLimit(target));
|
|
209
317
|
logger_1.logger.info(`Ranking API returned ${illusts.length} illustration(s)`);
|
|
210
318
|
return illusts;
|
|
211
319
|
}
|
|
@@ -234,9 +342,20 @@ class IllustrationTargetHandler {
|
|
|
234
342
|
const { downloaded, skipped, alreadyDownloaded, filteredOut } = result;
|
|
235
343
|
const targetLimit = target.limit || 10;
|
|
236
344
|
const tagForLog = (0, target_label_1.getTargetLabel)(target);
|
|
237
|
-
|
|
345
|
+
// The bounded scan produced its own explicit verdict when it attempted
|
|
346
|
+
// candidates or pre-filtered some, so the legacy "zero downloads" reporter
|
|
347
|
+
// is only the fallback for a scan that had nothing at all to look at.
|
|
348
|
+
const scanOwnsVerdict = result.scan.attempted > 0 || result.scan.skipped.length > 0;
|
|
349
|
+
if (downloaded === 0 && targetLimit > 0 && !scanOwnsVerdict) {
|
|
238
350
|
this.handleZeroDownloads(alreadyDownloaded, skipped, filteredOut, totalFound, targetLimit, tagForLog, mode, result.skipDetails);
|
|
239
351
|
}
|
|
352
|
+
if (scanOwnsVerdict && downloaded > 0) {
|
|
353
|
+
// The explicit success sentence the operator needs: which work was
|
|
354
|
+
// submitted, and how many unusable candidates were skipped to reach it.
|
|
355
|
+
logger_1.logger.info(`Candidate scan verdict for ${tagForLog}: submitted after skipping ` +
|
|
356
|
+
`${result.scan.skipped.length} candidate(s) of ${result.scan.attempted} attempted ` +
|
|
357
|
+
`(bound ${result.scan.bound})`);
|
|
358
|
+
}
|
|
240
359
|
if (downloaded > 0 && downloaded < targetLimit * 0.5 && skipped > 0) {
|
|
241
360
|
logger_1.logger.warn(`Only downloaded ${downloaded} out of ${targetLimit} requested illustration(s). ${skipped} illustration(s) were skipped due to errors.`);
|
|
242
361
|
}
|
|
@@ -295,11 +414,28 @@ class IllustrationTargetHandler {
|
|
|
295
414
|
try {
|
|
296
415
|
if (this.database.hasDownloaded(String(illustId), 'illustration')) {
|
|
297
416
|
logger_1.logger.info(`Illustration ${illustId} already downloaded, skipping`);
|
|
417
|
+
// A pinned single-work target has no candidate list to advance to, but
|
|
418
|
+
// the reason is still recorded as a candidate skip so the terminal
|
|
419
|
+
// outcome names it instead of reporting an unexplained no-op.
|
|
420
|
+
this.scan = {
|
|
421
|
+
bound: 1,
|
|
422
|
+
attempted: 0,
|
|
423
|
+
skipped: [
|
|
424
|
+
{ code: 'duplicate', workId: String(illustId), reason: 'already in download history' },
|
|
425
|
+
],
|
|
426
|
+
outages: [],
|
|
427
|
+
};
|
|
298
428
|
return;
|
|
299
429
|
}
|
|
300
430
|
const detail = await this.client.getIllustration(illustId);
|
|
301
431
|
// Use the detail directly as it's already a PixivIllust
|
|
302
|
-
await this.downloadAndDeliver(detail, `illust-${illustId}`, target);
|
|
432
|
+
const attempt = await this.downloadAndDeliver(detail, `illust-${illustId}`, target);
|
|
433
|
+
this.scan = {
|
|
434
|
+
bound: 1,
|
|
435
|
+
attempted: 1,
|
|
436
|
+
skipped: attempt.kind === 'skipped' ? [attempt.skip] : [],
|
|
437
|
+
outages: [],
|
|
438
|
+
};
|
|
303
439
|
logger_1.logger.info(`Successfully downloaded illustration ${illustId}`);
|
|
304
440
|
}
|
|
305
441
|
catch (error) {
|
|
@@ -325,6 +461,7 @@ class IllustrationTargetHandler {
|
|
|
325
461
|
return;
|
|
326
462
|
}
|
|
327
463
|
const result = await this.pipeline.run(illusts, target, 'illustration', (illust, tag) => this.downloadAndDeliver(illust, tag, target));
|
|
464
|
+
this.scan = result.scan;
|
|
328
465
|
this.handleDownloadResult(result, target, 'user', illusts.length);
|
|
329
466
|
}
|
|
330
467
|
catch (error) {
|
|
@@ -405,7 +542,16 @@ class IllustrationTargetHandler {
|
|
|
405
542
|
}
|
|
406
543
|
try {
|
|
407
544
|
const detail = await this.client.getIllustration(illustId);
|
|
408
|
-
await this.downloadAndDeliver(detail, displayTag, target);
|
|
545
|
+
const attempt = await this.downloadAndDeliver(detail, displayTag, target);
|
|
546
|
+
if (attempt.kind === 'skipped') {
|
|
547
|
+
// A cell that already owns a work cannot advance to another candidate —
|
|
548
|
+
// its identity is fixed. An already-delivered locked work is therefore a
|
|
549
|
+
// terminal business duplicate for THIS cell (it published nothing new),
|
|
550
|
+
// not a candidate skip and not an empty candidate list.
|
|
551
|
+
this.outcomes.push(attempt.skip.code === 'duplicate'
|
|
552
|
+
? { kind: 'duplicate', workId: lockedWorkId, reason: attempt.skip.reason }
|
|
553
|
+
: { kind: 'failed', retryable: false, error: `LOCKED_WORK_UNAVAILABLE: ${attempt.skip.reason}` });
|
|
554
|
+
}
|
|
409
555
|
}
|
|
410
556
|
catch (error) {
|
|
411
557
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -423,6 +569,11 @@ class IllustrationTargetHandler {
|
|
|
423
569
|
/**
|
|
424
570
|
* Process ONE candidate work for this cell.
|
|
425
571
|
*
|
|
572
|
+
* Returns what happened to THIS candidate, so the pipeline can advance to the
|
|
573
|
+
* next one when it was unusable. Throwing is reserved for JOB-level failures
|
|
574
|
+
* (dead database / dead token / dead delivery provider): a candidate-level
|
|
575
|
+
* failure is reported as a `skipped` attempt and never as a job verdict.
|
|
576
|
+
*
|
|
426
577
|
* The cell is bound to `illust` BEFORE any side effect: it is the binding, not
|
|
427
578
|
* the candidate list, that decides what a later recovery resumes. The binding
|
|
428
579
|
* is only rolled back when the attempt produced no artifact at all, so in-run
|
|
@@ -443,7 +594,14 @@ class IllustrationTargetHandler {
|
|
|
443
594
|
targetId: execution.targetId,
|
|
444
595
|
boundWorkId: binding.workId,
|
|
445
596
|
});
|
|
446
|
-
return
|
|
597
|
+
return {
|
|
598
|
+
kind: 'skipped',
|
|
599
|
+
skip: {
|
|
600
|
+
code: 'duplicate',
|
|
601
|
+
workId,
|
|
602
|
+
reason: `cell already owns work ${binding.workId} (concurrent selection)`,
|
|
603
|
+
},
|
|
604
|
+
};
|
|
447
605
|
}
|
|
448
606
|
}
|
|
449
607
|
let artifact = null;
|
|
@@ -458,59 +616,91 @@ class IllustrationTargetHandler {
|
|
|
458
616
|
// Nothing was persisted, so the cell may still pick another candidate.
|
|
459
617
|
if (execution && !recovering)
|
|
460
618
|
execution.release(workId);
|
|
461
|
-
|
|
619
|
+
const failure = (0, TargetOutcome_1.classifyCandidateFailure)(error, workId);
|
|
620
|
+
if (failure.scope === 'job' || !(0, TargetOutcome_1.skipCandidateWithoutRetry)(failure.skip)) {
|
|
621
|
+
// Job-level outage, or a transient candidate failure: re-thrown so the
|
|
622
|
+
// queue's existing retry/backoff owns it and the scheduler records a JOB
|
|
623
|
+
// failure — never an exhausted candidate list.
|
|
624
|
+
throw error;
|
|
625
|
+
}
|
|
626
|
+
this.logError(error, `Candidate illustration ${workId} skipped (${failure.skip.code})`);
|
|
627
|
+
return { kind: 'skipped', skip: failure.skip };
|
|
462
628
|
}
|
|
463
629
|
if (!artifact) {
|
|
630
|
+
// Nothing was persisted: the downloader deliberately declined this
|
|
631
|
+
// candidate (over maxPageCount, AI metadata check, already on disk). That
|
|
632
|
+
// is a fact about the work, so the scan moves on instead of retrying it.
|
|
464
633
|
if (execution && !recovering)
|
|
465
634
|
execution.release(workId);
|
|
466
|
-
return
|
|
635
|
+
return {
|
|
636
|
+
kind: 'skipped',
|
|
637
|
+
skip: { code: 'filtered', workId, reason: 'downloader declined this candidate (no artifact)' },
|
|
638
|
+
};
|
|
467
639
|
}
|
|
468
|
-
//
|
|
469
|
-
//
|
|
470
|
-
|
|
640
|
+
// The artifact is durable, but this cell has NOT committed to it until the
|
|
641
|
+
// delivery below actually claims it. A candidate that turns out to be a
|
|
642
|
+
// duplicate must give the binding BACK, or `bind()` would refuse the next
|
|
643
|
+
// candidate and the scan could never advance — the whole point of this
|
|
644
|
+
// change. `releaseCellWork` itself refuses once the cell moved on
|
|
645
|
+
// (delivery_pending/submitted), so a committed identity stays stable.
|
|
646
|
+
const attempt = this.recordArtifactOutcome(artifact, target);
|
|
647
|
+
if (attempt.kind === 'skipped' && execution && !recovering) {
|
|
648
|
+
execution.release(workId);
|
|
649
|
+
}
|
|
650
|
+
return attempt;
|
|
471
651
|
}
|
|
472
652
|
/**
|
|
473
653
|
* Turn a downloaded artifact into the target's business outcome. In cache
|
|
474
654
|
* delivery mode the DeliveryService creates the durable intent atomically and
|
|
475
655
|
* the result is 'delivery_pending' (NOT submitted — the OutboxWorker confirms
|
|
476
656
|
* the ACK). Persistent/download-only runs are 'stored'.
|
|
657
|
+
*
|
|
658
|
+
* An ALREADY-DELIVERED work is returned as a candidate SKIP, never as a
|
|
659
|
+
* target outcome: the run must try the next candidate instead of ending the
|
|
660
|
+
* slot as a `duplicate`. The delivery idempotency ledger is what makes the
|
|
661
|
+
* second concurrent worker lose this race instead of double-submitting.
|
|
477
662
|
*/
|
|
478
663
|
recordArtifactOutcome(artifact, target) {
|
|
479
664
|
const isDelivery = target.storageMode === 'cache' && target.delivery?.target?.trim();
|
|
480
665
|
if (!isDelivery || !this.deliveryService) {
|
|
481
666
|
this.outcomes.push({ kind: 'stored', workId: artifact.pixivId, workType: artifact.type });
|
|
482
|
-
return;
|
|
667
|
+
return { kind: 'selected', workId: artifact.pixivId, workType: artifact.type };
|
|
483
668
|
}
|
|
484
669
|
// Pre-lock delivery dedupe (after selection): if the ledger already knows it,
|
|
485
670
|
// that is a confirmed fact, not a new submission.
|
|
486
671
|
const slotId = target.delivery?.executionContext?.slotId;
|
|
487
672
|
if (this.deliveryService.isAlreadyDelivered(target.delivery.target, artifact.type, artifact.pixivId)) {
|
|
488
|
-
|
|
489
|
-
|
|
673
|
+
return {
|
|
674
|
+
kind: 'skipped',
|
|
675
|
+
skip: {
|
|
676
|
+
code: 'duplicate',
|
|
677
|
+
workId: artifact.pixivId,
|
|
678
|
+
reason: 'already delivered to target (delivery ledger)',
|
|
679
|
+
},
|
|
680
|
+
};
|
|
490
681
|
}
|
|
491
682
|
const res = this.deliveryService.enqueue(artifact, target, {
|
|
492
683
|
slotId,
|
|
493
684
|
fields: target.delivery?.fields,
|
|
494
|
-
extraContext:
|
|
685
|
+
extraContext: (0, deliveryContext_1.deliveryContextFields)(target),
|
|
495
686
|
});
|
|
496
687
|
if (res.duplicate) {
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
688
|
+
return {
|
|
689
|
+
kind: 'skipped',
|
|
690
|
+
skip: {
|
|
691
|
+
code: 'duplicate',
|
|
692
|
+
workId: artifact.pixivId,
|
|
693
|
+
reason: 'already delivered to target (idempotency ledger)',
|
|
694
|
+
},
|
|
695
|
+
};
|
|
501
696
|
}
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
triggerSource: ec?.executionContext?.triggerSource,
|
|
510
|
-
slotId: ec?.slotContext?.slotId ?? ec?.executionContext?.slotId,
|
|
511
|
-
slotName: ec?.slotContext?.slotName ?? ec?.executionContext?.slotName,
|
|
512
|
-
slotDate: ec?.slotContext?.slotDate ?? ec?.executionContext?.slotDate,
|
|
513
|
-
};
|
|
697
|
+
this.outcomes.push({
|
|
698
|
+
kind: 'delivery_pending',
|
|
699
|
+
workId: artifact.pixivId,
|
|
700
|
+
workType: artifact.type,
|
|
701
|
+
deliveryId: res.deliveryId,
|
|
702
|
+
});
|
|
703
|
+
return { kind: 'selected', workId: artifact.pixivId, workType: artifact.type };
|
|
514
704
|
}
|
|
515
705
|
}
|
|
516
706
|
exports.IllustrationTargetHandler = IllustrationTargetHandler;
|
|
@@ -17,6 +17,29 @@ export declare class NovelTargetHandler {
|
|
|
17
17
|
private readonly topicPipelineFactory?;
|
|
18
18
|
private readonly deliveryService?;
|
|
19
19
|
private outcomes;
|
|
20
|
+
/**
|
|
21
|
+
* Candidate scan of the last pipeline.run() of this handle() call: how many
|
|
22
|
+
* candidates were attempted, which were skipped and why, and whether any
|
|
23
|
+
* job-level outage appeared. This is what turns an empty result into an
|
|
24
|
+
* explicit verdict instead of an ambiguous "completed".
|
|
25
|
+
*/
|
|
26
|
+
private scan;
|
|
27
|
+
/**
|
|
28
|
+
* Global candidate-scan bound (`download.candidateScanLimit`), supplied by
|
|
29
|
+
* DownloadManager. This is what lets the FETCH stage ask for more than one
|
|
30
|
+
* candidate: a scheduled one-post-per-slot target has `limit: 1`, and asking
|
|
31
|
+
* the ranking API for exactly one work is the reason a single duplicate used
|
|
32
|
+
* to be unfixable downstream.
|
|
33
|
+
*/
|
|
34
|
+
private defaultCandidateScanLimit?;
|
|
35
|
+
/** Publish the global candidate-scan bound for the fetch stage. */
|
|
36
|
+
setDefaultCandidateScanLimit(limit: number | undefined): void;
|
|
37
|
+
/**
|
|
38
|
+
* How many candidates to FETCH so the bounded scan has something to choose
|
|
39
|
+
* from. Same rule the planner applies to the attempt window, so fetch and
|
|
40
|
+
* scan agree on one bound.
|
|
41
|
+
*/
|
|
42
|
+
private candidateFetchLimit;
|
|
20
43
|
/**
|
|
21
44
|
* Cell identity for this handle() call. Set only for a single-work cell of a
|
|
22
45
|
* scheduled occurrence; null for ad-hoc runs and N-works-per-run targets, which
|
|
@@ -25,6 +48,16 @@ export declare class NovelTargetHandler {
|
|
|
25
48
|
private execution;
|
|
26
49
|
constructor(client: IPixivClient, database: IDatabase, rankingService: RankingService, pipeline: DownloadPipeline, novelDownloader: NovelDownloader, topicPipelineFactory?: TopicPipelineFactory | undefined, deliveryService?: DeliveryService | undefined);
|
|
27
50
|
handle(target: TargetConfig, execution?: TargetExecutionContext): Promise<TargetOutcome>;
|
|
51
|
+
/**
|
|
52
|
+
* Reduce the outcomes collected while processing one target to one verdict,
|
|
53
|
+
* including the bounded-scan bookkeeping.
|
|
54
|
+
*
|
|
55
|
+
* A `duplicate` is deliberately NOT a target verdict for a scan: a duplicate
|
|
56
|
+
* is a CANDIDATE problem (skip it and try the next), which is what the scan
|
|
57
|
+
* already did. Returning it here is the bug that made a scheduled slot report
|
|
58
|
+
* success after submitting nothing. It remains a verdict only for the
|
|
59
|
+
* single-work RECOVERY path, whose cell identity is fixed.
|
|
60
|
+
*/
|
|
28
61
|
private summarize;
|
|
29
62
|
private classifyError;
|
|
30
63
|
private fetchNovels;
|
|
@@ -55,11 +88,25 @@ export declare class NovelTargetHandler {
|
|
|
55
88
|
/**
|
|
56
89
|
* Process ONE candidate work for this cell.
|
|
57
90
|
*
|
|
91
|
+
* Returns what happened to THIS candidate, so the pipeline can advance to the
|
|
92
|
+
* next one when it was unusable. Throwing is reserved for JOB-level failures
|
|
93
|
+
* (dead database / dead token / dead delivery provider): a candidate-level
|
|
94
|
+
* failure is reported as a `skipped` attempt and never as a job verdict.
|
|
95
|
+
*
|
|
58
96
|
* The cell is bound to `novel` BEFORE any side effect: it is the binding, not
|
|
59
97
|
* the candidate list, that decides what a later recovery resumes. The binding
|
|
60
98
|
* is only rolled back when the attempt produced no artifact at all, so in-run
|
|
61
99
|
* backfill still works for a cell that has committed to nothing.
|
|
62
100
|
*/
|
|
63
101
|
private downloadAndDeliver;
|
|
102
|
+
/**
|
|
103
|
+
* Turn a downloaded novel artifact into the target's business outcome.
|
|
104
|
+
*
|
|
105
|
+
* An ALREADY-DELIVERED work is returned as a candidate SKIP, never as a
|
|
106
|
+
* target outcome: the run must try the next candidate instead of ending the
|
|
107
|
+
* slot as a `duplicate`. The delivery idempotency ledger is what makes the
|
|
108
|
+
* second concurrent worker lose this race instead of double-submitting.
|
|
109
|
+
*/
|
|
110
|
+
private recordArtifactOutcome;
|
|
64
111
|
}
|
|
65
112
|
//# sourceMappingURL=NovelTargetHandler.d.ts.map
|