pixivflow 2.19.4 → 2.19.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config/defaults.d.ts +1 -0
- package/dist/config/defaults.js +4 -0
- package/dist/config/environment.js +19 -0
- package/dist/config/types.d.ts +21 -0
- package/dist/config/validation.js +15 -0
- package/dist/delivery/DeliveryService.d.ts +9 -0
- package/dist/delivery/DeliveryService.js +11 -0
- package/dist/download/DownloadManager.js +12 -1
- package/dist/download/handlers/IllustrationTargetHandler.d.ts +42 -1
- package/dist/download/handlers/IllustrationTargetHandler.js +238 -37
- package/dist/download/handlers/NovelTargetHandler.d.ts +47 -0
- package/dist/download/handlers/NovelTargetHandler.js +211 -27
- package/dist/download/pipeline/DownloadPipeline.d.ts +22 -3
- package/dist/download/pipeline/DownloadPipeline.js +109 -92
- package/dist/download/plan/DownloadPlanner.d.ts +47 -1
- package/dist/download/plan/DownloadPlanner.js +105 -15
- package/dist/package.json +1 -1
- package/dist/scheduler/TargetOutcome.d.ts +195 -2
- package/dist/scheduler/TargetOutcome.js +202 -3
- package/dist/storage/repositories/DeliveryRepository.d.ts +12 -0
- package/dist/storage/repositories/DeliveryRepository.js +18 -1
- package/dist/version.js +1 -1
- package/dist/webui/package.json +1 -1
- package/package.json +1 -1
|
@@ -5,8 +5,10 @@ const logger_1 = require("../../logger");
|
|
|
5
5
|
const pixiv_date_utils_1 = require("../../utils/pixiv-date-utils");
|
|
6
6
|
const errors_1 = require("../../utils/errors");
|
|
7
7
|
const pixiv_utils_1 = require("../../utils/pixiv-utils");
|
|
8
|
+
const TargetOutcome_1 = require("../../scheduler/TargetOutcome");
|
|
8
9
|
const WorkIdentity_1 = require("../../scheduler/WorkIdentity");
|
|
9
10
|
const target_label_1 = require("../../utils/target-label");
|
|
11
|
+
const DownloadPlanner_1 = require("../plan/DownloadPlanner");
|
|
10
12
|
class IllustrationTargetHandler {
|
|
11
13
|
client;
|
|
12
14
|
database;
|
|
@@ -17,6 +19,34 @@ class IllustrationTargetHandler {
|
|
|
17
19
|
deliveryService;
|
|
18
20
|
/** Outcomes produced during this handle() call (deliveries + terminal non-matches). */
|
|
19
21
|
outcomes = [];
|
|
22
|
+
/**
|
|
23
|
+
* Candidate scan of the last pipeline.run() of this handle() call: how many
|
|
24
|
+
* candidates were attempted, which were skipped and why, and whether any
|
|
25
|
+
* job-level outage appeared. This is what turns an empty result into an
|
|
26
|
+
* explicit verdict instead of an ambiguous "completed".
|
|
27
|
+
*/
|
|
28
|
+
scan = null;
|
|
29
|
+
/**
|
|
30
|
+
* Global candidate-scan bound (`download.candidateScanLimit`), supplied by
|
|
31
|
+
* DownloadManager. This is what lets the FETCH stage ask for more than one
|
|
32
|
+
* candidate: a scheduled one-post-per-slot target has `limit: 1`, and asking
|
|
33
|
+
* the ranking API for exactly one work is the reason a single duplicate used
|
|
34
|
+
* to be unfixable downstream.
|
|
35
|
+
*/
|
|
36
|
+
defaultCandidateScanLimit;
|
|
37
|
+
/** Publish the global candidate-scan bound for the fetch stage. */
|
|
38
|
+
setDefaultCandidateScanLimit(limit) {
|
|
39
|
+
this.defaultCandidateScanLimit = limit;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* How many candidates to FETCH so the bounded scan has something to choose
|
|
43
|
+
* from. Same rule the planner applies to the attempt window, so fetch and
|
|
44
|
+
* scan agree on one bound.
|
|
45
|
+
*/
|
|
46
|
+
candidateFetchLimit(target) {
|
|
47
|
+
const targetLimit = target.limit && target.limit > 0 ? target.limit : 10;
|
|
48
|
+
return Math.max(targetLimit, (0, DownloadPlanner_1.resolveCandidateScanLimit)(target, this.defaultCandidateScanLimit));
|
|
49
|
+
}
|
|
20
50
|
/**
|
|
21
51
|
* Cell identity for this handle() call. Set only for a single-work cell of a
|
|
22
52
|
* scheduled occurrence; null for ad-hoc runs and N-works-per-run targets, which
|
|
@@ -34,6 +64,7 @@ class IllustrationTargetHandler {
|
|
|
34
64
|
}
|
|
35
65
|
async handle(target, execution) {
|
|
36
66
|
this.outcomes = [];
|
|
67
|
+
this.scan = null;
|
|
37
68
|
this.execution = execution && (0, WorkIdentity_1.isSingleWorkCell)(target) ? execution : null;
|
|
38
69
|
// A cell that already owns a work is in RECOVERY, not in a new selection.
|
|
39
70
|
// Crash/shutdown recovery is not an intentional second run: running the
|
|
@@ -61,6 +92,7 @@ class IllustrationTargetHandler {
|
|
|
61
92
|
}
|
|
62
93
|
const illusts = await this.fetchIllustrations(target, mode);
|
|
63
94
|
const result = await this.pipeline.run(illusts, target, 'illustration', (illust, tag) => this.downloadAndDeliver(illust, tag, target));
|
|
95
|
+
this.scan = result.scan;
|
|
64
96
|
this.handleDownloadResult(result, target, mode, illusts.length);
|
|
65
97
|
return this.summarize(target);
|
|
66
98
|
}
|
|
@@ -68,40 +100,89 @@ class IllustrationTargetHandler {
|
|
|
68
100
|
return this.classifyError(error, displayTag, mode, target);
|
|
69
101
|
}
|
|
70
102
|
}
|
|
71
|
-
/**
|
|
103
|
+
/**
|
|
104
|
+
* Reduce the outcomes collected while processing one target to one cell
|
|
105
|
+
* result, including the bounded-scan bookkeeping.
|
|
106
|
+
*
|
|
107
|
+
* A `duplicate` is deliberately NOT a target verdict here: a duplicate is a
|
|
108
|
+
* CANDIDATE problem (skip it and try the next candidate), which is what the
|
|
109
|
+
* scan already did. Returning it as the target outcome is the bug that made a
|
|
110
|
+
* scheduled slot report success after submitting nothing.
|
|
111
|
+
*/
|
|
72
112
|
summarize(target) {
|
|
113
|
+
const scan = this.scan ?? undefined;
|
|
73
114
|
const submitted = this.outcomes.find((o) => o.kind === 'submitted');
|
|
74
115
|
if (submitted)
|
|
75
|
-
return submitted;
|
|
116
|
+
return scan ? { ...submitted, scan } : submitted;
|
|
76
117
|
const stored = this.outcomes.find((o) => o.kind === 'stored');
|
|
77
118
|
if (stored)
|
|
78
|
-
return stored;
|
|
119
|
+
return scan ? { ...stored, scan } : stored;
|
|
79
120
|
const pending = this.outcomes.find((o) => o.kind === 'delivery_pending');
|
|
80
121
|
if (pending)
|
|
81
|
-
return pending;
|
|
82
|
-
const
|
|
83
|
-
if (
|
|
84
|
-
return
|
|
85
|
-
const
|
|
86
|
-
if (
|
|
87
|
-
return
|
|
88
|
-
|
|
122
|
+
return scan ? { ...pending, scan } : pending;
|
|
123
|
+
const alreadyFailed = this.outcomes.find((o) => o.kind === 'failed');
|
|
124
|
+
if (alreadyFailed)
|
|
125
|
+
return scan ? { ...alreadyFailed, scan } : alreadyFailed;
|
|
126
|
+
const terminalDuplicate = this.outcomes.find((o) => o.kind === 'duplicate');
|
|
127
|
+
if (terminalDuplicate)
|
|
128
|
+
return scan ? { ...terminalDuplicate, scan } : terminalDuplicate;
|
|
129
|
+
// A job-level outage is never an empty candidate list. Failing here keeps
|
|
130
|
+
// the existing retry/backoff semantics: the scheduler retries the job on a
|
|
131
|
+
// later trigger instead of recording a clean no-op.
|
|
132
|
+
if (scan && scan.outages.length > 0) {
|
|
133
|
+
return {
|
|
134
|
+
kind: 'failed',
|
|
135
|
+
retryable: true,
|
|
136
|
+
error: `job-level outage while scanning candidates: ${scan.outages.join(', ')}`,
|
|
137
|
+
scan,
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
if (scan && (0, TargetOutcome_1.hasTransientFailure)(scan)) {
|
|
141
|
+
return {
|
|
142
|
+
kind: 'failed',
|
|
143
|
+
retryable: true,
|
|
144
|
+
error: `candidate scan hit transient infrastructure failures ` +
|
|
145
|
+
`(${scan.skipped.filter((s) => s.retryable).length} of ${scan.attempted} attempted)`,
|
|
146
|
+
scan,
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
if (scan && (scan.attempted > 0 || scan.skipped.length > 0)) {
|
|
150
|
+
return { kind: 'no_candidate', reason: (0, TargetOutcome_1.noEligibleCandidateText)(scan), scan };
|
|
151
|
+
}
|
|
152
|
+
return {
|
|
153
|
+
kind: 'no_candidate',
|
|
154
|
+
reason: 'no matching illustration after filtering/dedupe',
|
|
155
|
+
...(scan ? { scan } : {}),
|
|
156
|
+
};
|
|
89
157
|
}
|
|
90
158
|
classifyError(error, displayTag, mode, target) {
|
|
91
159
|
const message = error instanceof Error ? error.message : String(error);
|
|
160
|
+
const scan = this.scan ?? undefined;
|
|
161
|
+
// A hard job-level outage is named as such and is never recorded as a
|
|
162
|
+
// no-candidate business outcome, whatever its message happens to look like.
|
|
163
|
+
const outage = (0, TargetOutcome_1.classifyJobLevelOutage)(error);
|
|
92
164
|
this.database.logExecution(displayTag, 'illustration', 'failed', message);
|
|
93
165
|
logger_1.logger.error(`Illustration ${mode === 'ranking' ? 'ranking' : 'tag'} ${displayTag} failed`, {
|
|
94
166
|
error: message,
|
|
95
167
|
errorType: error instanceof Error ? error.constructor.name : typeof error,
|
|
168
|
+
...(outage ? { jobLevelOutage: outage } : {}),
|
|
96
169
|
});
|
|
170
|
+
if (outage) {
|
|
171
|
+
return {
|
|
172
|
+
kind: 'failed',
|
|
173
|
+
retryable: true,
|
|
174
|
+
error: `job-level outage (${outage}): ${message}`,
|
|
175
|
+
...(scan ? { scan } : {}),
|
|
176
|
+
};
|
|
177
|
+
}
|
|
97
178
|
// Explicit no-candidate signals are business outcomes, not failures.
|
|
98
179
|
if (/no matching|all .*filtered|no_candidate/i.test(message)) {
|
|
99
|
-
return { kind: 'no_candidate', reason: message };
|
|
180
|
+
return { kind: 'no_candidate', reason: message, ...(scan ? { scan } : {}) };
|
|
100
181
|
}
|
|
101
182
|
// Network/transient => retryable so the SAME work resumes on next trigger.
|
|
102
183
|
const retryable = (0, errors_1.isRetryableNetworkError)(error) ||
|
|
103
184
|
(error instanceof Error && /timeout|econn|enotfound|etimed|429|5\d\d/i.test(error.message));
|
|
104
|
-
return { kind: 'failed', retryable, error: message };
|
|
185
|
+
return { kind: 'failed', retryable, error: message, ...(scan ? { scan } : {}) };
|
|
105
186
|
}
|
|
106
187
|
async fetchIllustrations(target, mode) {
|
|
107
188
|
if (mode === 'topic') {
|
|
@@ -150,15 +231,37 @@ class IllustrationTargetHandler {
|
|
|
150
231
|
}
|
|
151
232
|
const illusts = await this.fetchTopicIllustrations(attemptTarget);
|
|
152
233
|
const result = await this.pipeline.run(illusts, attemptTarget, 'illustration', (illust, tag) => this.downloadAndDeliver(illust, tag, attemptTarget));
|
|
234
|
+
// The lookback loop is one bounded scan: accumulate every day's
|
|
235
|
+
// skips/outages so the final verdict covers all candidates attempted.
|
|
236
|
+
this.scan = (0, TargetOutcome_1.mergeScanSummaries)(this.scan, result.scan);
|
|
153
237
|
if (result.downloaded > 0) {
|
|
154
238
|
this.handleDownloadResult(result, target, 'topic', illusts.length);
|
|
155
239
|
return;
|
|
156
240
|
}
|
|
241
|
+
if (this.scan && this.scan.outages.length > 0) {
|
|
242
|
+
// A dead token / dead database / dead network is not "no matching
|
|
243
|
+
// illustration": stop looking back and let the job fail/retry.
|
|
244
|
+
return;
|
|
245
|
+
}
|
|
157
246
|
}
|
|
158
|
-
const
|
|
247
|
+
const scan = this.scan;
|
|
248
|
+
// An explicit no-eligible-candidate verdict needs something to have been
|
|
249
|
+
// considered. When the scan is empty (nothing surfaced at all) the richer
|
|
250
|
+
// existing message — which names the days actually checked — is the honest
|
|
251
|
+
// one, so it is kept.
|
|
252
|
+
const considered = Boolean(scan && (scan.attempted > 0 || scan.skipped.length > 0));
|
|
253
|
+
const message = considered
|
|
254
|
+
? `${(0, TargetOutcome_1.noEligibleCandidateText)(scan)} (checked ${checkedDays.join(', ')})`
|
|
255
|
+
: `No matching illustrations found after checking ${checkedDays.length} day(s): ${checkedDays.join(', ')}`;
|
|
256
|
+
// A bounded, exhausted scan IS the success path for a no-op day: nothing
|
|
257
|
+
// was eligible and nothing was submitted. Recorded as an explicit verdict.
|
|
159
258
|
this.database.logExecution(displayTag, 'illustration', 'success', message);
|
|
160
|
-
logger_1.logger.warn(`Illustration topic ${displayTag} produced no matching result`, { checkedDays });
|
|
161
|
-
this.outcomes.push({
|
|
259
|
+
logger_1.logger.warn(`Illustration topic ${displayTag} produced no matching result`, { checkedDays, message });
|
|
260
|
+
this.outcomes.push({
|
|
261
|
+
kind: 'no_candidate',
|
|
262
|
+
reason: message,
|
|
263
|
+
...(scan ? { scan } : {}),
|
|
264
|
+
});
|
|
162
265
|
}
|
|
163
266
|
resolveTopicDay(target) {
|
|
164
267
|
return target.date === 'TODAY'
|
|
@@ -189,12 +292,14 @@ class IllustrationTargetHandler {
|
|
|
189
292
|
endDate: rankingDate,
|
|
190
293
|
limit: Math.max(targetLimit * 20, 100),
|
|
191
294
|
};
|
|
295
|
+
const fetchLimit = this.candidateFetchLimit(target);
|
|
192
296
|
let illusts = await this.client.searchIllustrations(searchTarget);
|
|
193
297
|
logger_1.logger.info(`Found ${illusts.length} illustration(s) for ${rankingDate}`);
|
|
194
|
-
this.sortByPopularityAndLog(illusts,
|
|
195
|
-
if (illusts.length >
|
|
196
|
-
illusts = illusts.slice(0,
|
|
197
|
-
logger_1.logger.info(`Selected top ${illusts.length} illustration(s) by popularity`
|
|
298
|
+
this.sortByPopularityAndLog(illusts, fetchLimit, 'illustration');
|
|
299
|
+
if (illusts.length > fetchLimit) {
|
|
300
|
+
illusts = illusts.slice(0, fetchLimit);
|
|
301
|
+
logger_1.logger.info(`Selected top ${illusts.length} illustration(s) by popularity ` +
|
|
302
|
+
`(to fill ${targetLimit}, candidate scan bound ${fetchLimit})`);
|
|
198
303
|
}
|
|
199
304
|
return illusts;
|
|
200
305
|
}
|
|
@@ -205,7 +310,9 @@ class IllustrationTargetHandler {
|
|
|
205
310
|
rankingDate = (0, pixiv_date_utils_1.getYesterdayDate)();
|
|
206
311
|
}
|
|
207
312
|
logger_1.logger.info(`Fetching ranking illustrations (mode: ${rankingMode}, date: ${rankingDate})`);
|
|
208
|
-
|
|
313
|
+
// Ask for the whole bounded scan window, not `limit`: the planner then
|
|
314
|
+
// narrows it to the candidates this run may attempt.
|
|
315
|
+
const illusts = await this.rankingService.getRankingIllustrationsWithFallback(rankingMode, rankingDate, this.candidateFetchLimit(target));
|
|
209
316
|
logger_1.logger.info(`Ranking API returned ${illusts.length} illustration(s)`);
|
|
210
317
|
return illusts;
|
|
211
318
|
}
|
|
@@ -234,9 +341,20 @@ class IllustrationTargetHandler {
|
|
|
234
341
|
const { downloaded, skipped, alreadyDownloaded, filteredOut } = result;
|
|
235
342
|
const targetLimit = target.limit || 10;
|
|
236
343
|
const tagForLog = (0, target_label_1.getTargetLabel)(target);
|
|
237
|
-
|
|
344
|
+
// The bounded scan produced its own explicit verdict when it attempted
|
|
345
|
+
// candidates or pre-filtered some, so the legacy "zero downloads" reporter
|
|
346
|
+
// is only the fallback for a scan that had nothing at all to look at.
|
|
347
|
+
const scanOwnsVerdict = result.scan.attempted > 0 || result.scan.skipped.length > 0;
|
|
348
|
+
if (downloaded === 0 && targetLimit > 0 && !scanOwnsVerdict) {
|
|
238
349
|
this.handleZeroDownloads(alreadyDownloaded, skipped, filteredOut, totalFound, targetLimit, tagForLog, mode, result.skipDetails);
|
|
239
350
|
}
|
|
351
|
+
if (scanOwnsVerdict && downloaded > 0) {
|
|
352
|
+
// The explicit success sentence the operator needs: which work was
|
|
353
|
+
// submitted, and how many unusable candidates were skipped to reach it.
|
|
354
|
+
logger_1.logger.info(`Candidate scan verdict for ${tagForLog}: submitted after skipping ` +
|
|
355
|
+
`${result.scan.skipped.length} candidate(s) of ${result.scan.attempted} attempted ` +
|
|
356
|
+
`(bound ${result.scan.bound})`);
|
|
357
|
+
}
|
|
240
358
|
if (downloaded > 0 && downloaded < targetLimit * 0.5 && skipped > 0) {
|
|
241
359
|
logger_1.logger.warn(`Only downloaded ${downloaded} out of ${targetLimit} requested illustration(s). ${skipped} illustration(s) were skipped due to errors.`);
|
|
242
360
|
}
|
|
@@ -295,11 +413,28 @@ class IllustrationTargetHandler {
|
|
|
295
413
|
try {
|
|
296
414
|
if (this.database.hasDownloaded(String(illustId), 'illustration')) {
|
|
297
415
|
logger_1.logger.info(`Illustration ${illustId} already downloaded, skipping`);
|
|
416
|
+
// A pinned single-work target has no candidate list to advance to, but
|
|
417
|
+
// the reason is still recorded as a candidate skip so the terminal
|
|
418
|
+
// outcome names it instead of reporting an unexplained no-op.
|
|
419
|
+
this.scan = {
|
|
420
|
+
bound: 1,
|
|
421
|
+
attempted: 0,
|
|
422
|
+
skipped: [
|
|
423
|
+
{ code: 'duplicate', workId: String(illustId), reason: 'already in download history' },
|
|
424
|
+
],
|
|
425
|
+
outages: [],
|
|
426
|
+
};
|
|
298
427
|
return;
|
|
299
428
|
}
|
|
300
429
|
const detail = await this.client.getIllustration(illustId);
|
|
301
430
|
// Use the detail directly as it's already a PixivIllust
|
|
302
|
-
await this.downloadAndDeliver(detail, `illust-${illustId}`, target);
|
|
431
|
+
const attempt = await this.downloadAndDeliver(detail, `illust-${illustId}`, target);
|
|
432
|
+
this.scan = {
|
|
433
|
+
bound: 1,
|
|
434
|
+
attempted: 1,
|
|
435
|
+
skipped: attempt.kind === 'skipped' ? [attempt.skip] : [],
|
|
436
|
+
outages: [],
|
|
437
|
+
};
|
|
303
438
|
logger_1.logger.info(`Successfully downloaded illustration ${illustId}`);
|
|
304
439
|
}
|
|
305
440
|
catch (error) {
|
|
@@ -325,6 +460,7 @@ class IllustrationTargetHandler {
|
|
|
325
460
|
return;
|
|
326
461
|
}
|
|
327
462
|
const result = await this.pipeline.run(illusts, target, 'illustration', (illust, tag) => this.downloadAndDeliver(illust, tag, target));
|
|
463
|
+
this.scan = result.scan;
|
|
328
464
|
this.handleDownloadResult(result, target, 'user', illusts.length);
|
|
329
465
|
}
|
|
330
466
|
catch (error) {
|
|
@@ -405,7 +541,16 @@ class IllustrationTargetHandler {
|
|
|
405
541
|
}
|
|
406
542
|
try {
|
|
407
543
|
const detail = await this.client.getIllustration(illustId);
|
|
408
|
-
await this.downloadAndDeliver(detail, displayTag, target);
|
|
544
|
+
const attempt = await this.downloadAndDeliver(detail, displayTag, target);
|
|
545
|
+
if (attempt.kind === 'skipped') {
|
|
546
|
+
// A cell that already owns a work cannot advance to another candidate —
|
|
547
|
+
// its identity is fixed. An already-delivered locked work is therefore a
|
|
548
|
+
// terminal business duplicate for THIS cell (it published nothing new),
|
|
549
|
+
// not a candidate skip and not an empty candidate list.
|
|
550
|
+
this.outcomes.push(attempt.skip.code === 'duplicate'
|
|
551
|
+
? { kind: 'duplicate', workId: lockedWorkId, reason: attempt.skip.reason }
|
|
552
|
+
: { kind: 'failed', retryable: false, error: `LOCKED_WORK_UNAVAILABLE: ${attempt.skip.reason}` });
|
|
553
|
+
}
|
|
409
554
|
}
|
|
410
555
|
catch (error) {
|
|
411
556
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -423,6 +568,11 @@ class IllustrationTargetHandler {
|
|
|
423
568
|
/**
|
|
424
569
|
* Process ONE candidate work for this cell.
|
|
425
570
|
*
|
|
571
|
+
* Returns what happened to THIS candidate, so the pipeline can advance to the
|
|
572
|
+
* next one when it was unusable. Throwing is reserved for JOB-level failures
|
|
573
|
+
* (dead database / dead token / dead delivery provider): a candidate-level
|
|
574
|
+
* failure is reported as a `skipped` attempt and never as a job verdict.
|
|
575
|
+
*
|
|
426
576
|
* The cell is bound to `illust` BEFORE any side effect: it is the binding, not
|
|
427
577
|
* the candidate list, that decides what a later recovery resumes. The binding
|
|
428
578
|
* is only rolled back when the attempt produced no artifact at all, so in-run
|
|
@@ -443,7 +593,14 @@ class IllustrationTargetHandler {
|
|
|
443
593
|
targetId: execution.targetId,
|
|
444
594
|
boundWorkId: binding.workId,
|
|
445
595
|
});
|
|
446
|
-
return
|
|
596
|
+
return {
|
|
597
|
+
kind: 'skipped',
|
|
598
|
+
skip: {
|
|
599
|
+
code: 'duplicate',
|
|
600
|
+
workId,
|
|
601
|
+
reason: `cell already owns work ${binding.workId} (concurrent selection)`,
|
|
602
|
+
},
|
|
603
|
+
};
|
|
447
604
|
}
|
|
448
605
|
}
|
|
449
606
|
let artifact = null;
|
|
@@ -458,35 +615,68 @@ class IllustrationTargetHandler {
|
|
|
458
615
|
// Nothing was persisted, so the cell may still pick another candidate.
|
|
459
616
|
if (execution && !recovering)
|
|
460
617
|
execution.release(workId);
|
|
461
|
-
|
|
618
|
+
const failure = (0, TargetOutcome_1.classifyCandidateFailure)(error, workId);
|
|
619
|
+
if (failure.scope === 'job' || !(0, TargetOutcome_1.skipCandidateWithoutRetry)(failure.skip)) {
|
|
620
|
+
// Job-level outage, or a transient candidate failure: re-thrown so the
|
|
621
|
+
// queue's existing retry/backoff owns it and the scheduler records a JOB
|
|
622
|
+
// failure — never an exhausted candidate list.
|
|
623
|
+
throw error;
|
|
624
|
+
}
|
|
625
|
+
this.logError(error, `Candidate illustration ${workId} skipped (${failure.skip.code})`);
|
|
626
|
+
return { kind: 'skipped', skip: failure.skip };
|
|
462
627
|
}
|
|
463
628
|
if (!artifact) {
|
|
629
|
+
// Nothing was persisted: the downloader deliberately declined this
|
|
630
|
+
// candidate (over maxPageCount, AI metadata check, already on disk). That
|
|
631
|
+
// is a fact about the work, so the scan moves on instead of retrying it.
|
|
464
632
|
if (execution && !recovering)
|
|
465
633
|
execution.release(workId);
|
|
466
|
-
return
|
|
634
|
+
return {
|
|
635
|
+
kind: 'skipped',
|
|
636
|
+
skip: { code: 'filtered', workId, reason: 'downloader declined this candidate (no artifact)' },
|
|
637
|
+
};
|
|
467
638
|
}
|
|
468
|
-
//
|
|
469
|
-
//
|
|
470
|
-
|
|
639
|
+
// The artifact is durable, but this cell has NOT committed to it until the
|
|
640
|
+
// delivery below actually claims it. A candidate that turns out to be a
|
|
641
|
+
// duplicate must give the binding BACK, or `bind()` would refuse the next
|
|
642
|
+
// candidate and the scan could never advance — the whole point of this
|
|
643
|
+
// change. `releaseCellWork` itself refuses once the cell moved on
|
|
644
|
+
// (delivery_pending/submitted), so a committed identity stays stable.
|
|
645
|
+
const attempt = this.recordArtifactOutcome(artifact, target);
|
|
646
|
+
if (attempt.kind === 'skipped' && execution && !recovering) {
|
|
647
|
+
execution.release(workId);
|
|
648
|
+
}
|
|
649
|
+
return attempt;
|
|
471
650
|
}
|
|
472
651
|
/**
|
|
473
652
|
* Turn a downloaded artifact into the target's business outcome. In cache
|
|
474
653
|
* delivery mode the DeliveryService creates the durable intent atomically and
|
|
475
654
|
* the result is 'delivery_pending' (NOT submitted — the OutboxWorker confirms
|
|
476
655
|
* the ACK). Persistent/download-only runs are 'stored'.
|
|
656
|
+
*
|
|
657
|
+
* An ALREADY-DELIVERED work is returned as a candidate SKIP, never as a
|
|
658
|
+
* target outcome: the run must try the next candidate instead of ending the
|
|
659
|
+
* slot as a `duplicate`. The delivery idempotency ledger is what makes the
|
|
660
|
+
* second concurrent worker lose this race instead of double-submitting.
|
|
477
661
|
*/
|
|
478
662
|
recordArtifactOutcome(artifact, target) {
|
|
479
663
|
const isDelivery = target.storageMode === 'cache' && target.delivery?.target?.trim();
|
|
480
664
|
if (!isDelivery || !this.deliveryService) {
|
|
481
665
|
this.outcomes.push({ kind: 'stored', workId: artifact.pixivId, workType: artifact.type });
|
|
482
|
-
return;
|
|
666
|
+
return { kind: 'selected', workId: artifact.pixivId, workType: artifact.type };
|
|
483
667
|
}
|
|
484
668
|
// Pre-lock delivery dedupe (after selection): if the ledger already knows it,
|
|
485
669
|
// that is a confirmed fact, not a new submission.
|
|
486
670
|
const slotId = target.delivery?.executionContext?.slotId;
|
|
487
671
|
if (this.deliveryService.isAlreadyDelivered(target.delivery.target, artifact.type, artifact.pixivId)) {
|
|
488
|
-
|
|
489
|
-
|
|
672
|
+
return {
|
|
673
|
+
kind: 'skipped',
|
|
674
|
+
skip: {
|
|
675
|
+
code: 'duplicate',
|
|
676
|
+
workId: artifact.pixivId,
|
|
677
|
+
reason: 'already delivered to target (delivery ledger)',
|
|
678
|
+
},
|
|
679
|
+
};
|
|
490
680
|
}
|
|
491
681
|
const res = this.deliveryService.enqueue(artifact, target, {
|
|
492
682
|
slotId,
|
|
@@ -494,11 +684,22 @@ class IllustrationTargetHandler {
|
|
|
494
684
|
extraContext: this.executionContextFields(target),
|
|
495
685
|
});
|
|
496
686
|
if (res.duplicate) {
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
687
|
+
return {
|
|
688
|
+
kind: 'skipped',
|
|
689
|
+
skip: {
|
|
690
|
+
code: 'duplicate',
|
|
691
|
+
workId: artifact.pixivId,
|
|
692
|
+
reason: 'already delivered to target (idempotency ledger)',
|
|
693
|
+
},
|
|
694
|
+
};
|
|
501
695
|
}
|
|
696
|
+
this.outcomes.push({
|
|
697
|
+
kind: 'delivery_pending',
|
|
698
|
+
workId: artifact.pixivId,
|
|
699
|
+
workType: artifact.type,
|
|
700
|
+
deliveryId: res.deliveryId,
|
|
701
|
+
});
|
|
702
|
+
return { kind: 'selected', workId: artifact.pixivId, workType: artifact.type };
|
|
502
703
|
}
|
|
503
704
|
executionContextFields(target) {
|
|
504
705
|
const ec = target.delivery;
|
|
@@ -17,6 +17,29 @@ export declare class NovelTargetHandler {
|
|
|
17
17
|
private readonly topicPipelineFactory?;
|
|
18
18
|
private readonly deliveryService?;
|
|
19
19
|
private outcomes;
|
|
20
|
+
/**
|
|
21
|
+
* Candidate scan of the last pipeline.run() of this handle() call: how many
|
|
22
|
+
* candidates were attempted, which were skipped and why, and whether any
|
|
23
|
+
* job-level outage appeared. This is what turns an empty result into an
|
|
24
|
+
* explicit verdict instead of an ambiguous "completed".
|
|
25
|
+
*/
|
|
26
|
+
private scan;
|
|
27
|
+
/**
|
|
28
|
+
* Global candidate-scan bound (`download.candidateScanLimit`), supplied by
|
|
29
|
+
* DownloadManager. This is what lets the FETCH stage ask for more than one
|
|
30
|
+
* candidate: a scheduled one-post-per-slot target has `limit: 1`, and asking
|
|
31
|
+
* the ranking API for exactly one work is the reason a single duplicate used
|
|
32
|
+
* to be unfixable downstream.
|
|
33
|
+
*/
|
|
34
|
+
private defaultCandidateScanLimit?;
|
|
35
|
+
/** Publish the global candidate-scan bound for the fetch stage. */
|
|
36
|
+
setDefaultCandidateScanLimit(limit: number | undefined): void;
|
|
37
|
+
/**
|
|
38
|
+
* How many candidates to FETCH so the bounded scan has something to choose
|
|
39
|
+
* from. Same rule the planner applies to the attempt window, so fetch and
|
|
40
|
+
* scan agree on one bound.
|
|
41
|
+
*/
|
|
42
|
+
private candidateFetchLimit;
|
|
20
43
|
/**
|
|
21
44
|
* Cell identity for this handle() call. Set only for a single-work cell of a
|
|
22
45
|
* scheduled occurrence; null for ad-hoc runs and N-works-per-run targets, which
|
|
@@ -25,6 +48,16 @@ export declare class NovelTargetHandler {
|
|
|
25
48
|
private execution;
|
|
26
49
|
constructor(client: IPixivClient, database: IDatabase, rankingService: RankingService, pipeline: DownloadPipeline, novelDownloader: NovelDownloader, topicPipelineFactory?: TopicPipelineFactory | undefined, deliveryService?: DeliveryService | undefined);
|
|
27
50
|
handle(target: TargetConfig, execution?: TargetExecutionContext): Promise<TargetOutcome>;
|
|
51
|
+
/**
|
|
52
|
+
* Reduce the outcomes collected while processing one target to one verdict,
|
|
53
|
+
* including the bounded-scan bookkeeping.
|
|
54
|
+
*
|
|
55
|
+
* A `duplicate` is deliberately NOT a target verdict for a scan: a duplicate
|
|
56
|
+
* is a CANDIDATE problem (skip it and try the next), which is what the scan
|
|
57
|
+
* already did. Returning it here is the bug that made a scheduled slot report
|
|
58
|
+
* success after submitting nothing. It remains a verdict only for the
|
|
59
|
+
* single-work RECOVERY path, whose cell identity is fixed.
|
|
60
|
+
*/
|
|
28
61
|
private summarize;
|
|
29
62
|
private classifyError;
|
|
30
63
|
private fetchNovels;
|
|
@@ -55,11 +88,25 @@ export declare class NovelTargetHandler {
|
|
|
55
88
|
/**
|
|
56
89
|
* Process ONE candidate work for this cell.
|
|
57
90
|
*
|
|
91
|
+
* Returns what happened to THIS candidate, so the pipeline can advance to the
|
|
92
|
+
* next one when it was unusable. Throwing is reserved for JOB-level failures
|
|
93
|
+
* (dead database / dead token / dead delivery provider): a candidate-level
|
|
94
|
+
* failure is reported as a `skipped` attempt and never as a job verdict.
|
|
95
|
+
*
|
|
58
96
|
* The cell is bound to `novel` BEFORE any side effect: it is the binding, not
|
|
59
97
|
* the candidate list, that decides what a later recovery resumes. The binding
|
|
60
98
|
* is only rolled back when the attempt produced no artifact at all, so in-run
|
|
61
99
|
* backfill still works for a cell that has committed to nothing.
|
|
62
100
|
*/
|
|
63
101
|
private downloadAndDeliver;
|
|
102
|
+
/**
|
|
103
|
+
* Turn a downloaded novel artifact into the target's business outcome.
|
|
104
|
+
*
|
|
105
|
+
* An ALREADY-DELIVERED work is returned as a candidate SKIP, never as a
|
|
106
|
+
* target outcome: the run must try the next candidate instead of ending the
|
|
107
|
+
* slot as a `duplicate`. The delivery idempotency ledger is what makes the
|
|
108
|
+
* second concurrent worker lose this race instead of double-submitting.
|
|
109
|
+
*/
|
|
110
|
+
private recordArtifactOutcome;
|
|
64
111
|
}
|
|
65
112
|
//# sourceMappingURL=NovelTargetHandler.d.ts.map
|