pixivflow 2.19.4 → 2.19.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config/defaults.d.ts +1 -0
- package/dist/config/defaults.js +4 -0
- package/dist/config/environment.js +19 -0
- package/dist/config/types.d.ts +21 -0
- package/dist/config/validation.js +15 -0
- package/dist/delivery/DeliveryService.d.ts +9 -0
- package/dist/delivery/DeliveryService.js +11 -0
- package/dist/download/DownloadManager.js +12 -1
- package/dist/download/handlers/IllustrationTargetHandler.d.ts +42 -1
- package/dist/download/handlers/IllustrationTargetHandler.js +238 -37
- package/dist/download/handlers/NovelTargetHandler.d.ts +47 -0
- package/dist/download/handlers/NovelTargetHandler.js +211 -27
- package/dist/download/pipeline/DownloadPipeline.d.ts +22 -3
- package/dist/download/pipeline/DownloadPipeline.js +109 -92
- package/dist/download/plan/DownloadPlanner.d.ts +47 -1
- package/dist/download/plan/DownloadPlanner.js +105 -15
- package/dist/package.json +1 -1
- package/dist/scheduler/TargetOutcome.d.ts +195 -2
- package/dist/scheduler/TargetOutcome.js +202 -3
- package/dist/storage/repositories/DeliveryRepository.d.ts +12 -0
- package/dist/storage/repositories/DeliveryRepository.js +18 -1
- package/dist/version.js +1 -1
- package/dist/webui/package.json +1 -1
- package/package.json +1 -1
|
@@ -5,14 +5,138 @@
|
|
|
5
5
|
* submitted". Undefined / a 2xx HTTP response / a caught-and-swallowed error
|
|
6
6
|
* must NEVER be inferred as a successful submission. Every target produces one
|
|
7
7
|
* of these explicit outcomes; the Slot ledger maps it to a cell transition.
|
|
8
|
+
*
|
|
9
|
+
* The vocabulary has TWO levels, and conflating them is what let a duplicate
|
|
10
|
+
* candidate end a scheduled slot as "successful":
|
|
11
|
+
*
|
|
12
|
+
* - CANDIDATE level (`CandidateSkip`): this ONE candidate work was unusable —
|
|
13
|
+
* already delivered, deleted, private, wrong media, wrong language. A
|
|
14
|
+
* candidate problem is never a job verdict: the scan advances to the next
|
|
15
|
+
* candidate.
|
|
16
|
+
* - TARGET level (`TargetOutcome`, below): the verdict for the whole logical
|
|
17
|
+
* item after its bounded candidate scan, including the scan bookkeeping so
|
|
18
|
+
* "completed with nothing done" is impossible to report silently.
|
|
8
19
|
*/
|
|
9
20
|
export type WorkType = 'illustration' | 'novel';
|
|
21
|
+
/**
|
|
22
|
+
* Why ONE candidate work was skipped without producing a business result.
|
|
23
|
+
*
|
|
24
|
+
* Codes are deliberately outcome-shaped, not error-shaped: they say what the
|
|
25
|
+
* run should DO (try the next candidate), not which exception was raised.
|
|
26
|
+
*/
|
|
27
|
+
export type CandidateSkipCode =
|
|
28
|
+
/** Already delivered / already pending review for this target, or a lost
|
|
29
|
+
* idempotency race against a concurrent worker. Try the next candidate. */
|
|
30
|
+
'duplicate'
|
|
31
|
+
/** Gone: 404, deleted work, removed account. Permanent for this candidate. */
|
|
32
|
+
| 'deleted'
|
|
33
|
+
/** Private / R-18 without permission / 403 on THIS work. */
|
|
34
|
+
| 'access_denied'
|
|
35
|
+
/** Ugoira or novel form this build cannot process. */
|
|
36
|
+
| 'unsupported_media'
|
|
37
|
+
/** Missing or malformed metadata: no id, no pages, no image urls. */
|
|
38
|
+
| 'invalid_metadata'
|
|
39
|
+
/**
|
|
40
|
+
* The attempt failed for a reason that is NOT a fact about the work: timeout,
|
|
41
|
+
* connection reset, rate limit, 5xx, or an unclassifiable error. `retryable`
|
|
42
|
+
* is set, which means the candidate is handed back to the existing
|
|
43
|
+
* retry/backoff instead of being silently skipped — and a scan in which EVERY
|
|
44
|
+
* attempted candidate was transient is a JOB failure, not an empty candidate
|
|
45
|
+
* list.
|
|
46
|
+
*/
|
|
47
|
+
| 'unavailable'
|
|
48
|
+
/**
|
|
49
|
+
* Not usable for THIS run by the run's own rules: full-text language filter,
|
|
50
|
+
* over maxPageCount, AI-metadata check, or a candidate the downloader
|
|
51
|
+
* deliberately declined. Move on to the next candidate.
|
|
52
|
+
*/
|
|
53
|
+
| 'filtered';
|
|
54
|
+
export interface CandidateSkip {
|
|
55
|
+
code: CandidateSkipCode;
|
|
56
|
+
workId: string;
|
|
57
|
+
reason: string;
|
|
58
|
+
/**
|
|
59
|
+
* True when the cause is transient infrastructure rather than this work.
|
|
60
|
+
* One such candidate still means "try the next"; EVERY attempted candidate
|
|
61
|
+
* being retryable means the infrastructure — not the candidate list — is the
|
|
62
|
+
* problem, and the target must fail/retry instead of reporting "no eligible
|
|
63
|
+
* candidate".
|
|
64
|
+
*/
|
|
65
|
+
retryable?: boolean;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Infrastructure failure scoped to the WHOLE job, never to one candidate. These
|
|
69
|
+
* must abort/fail/retry the job; they must never be swallowed as "skip and try
|
|
70
|
+
* the next candidate" (that is the failure mode that burned scheduled slots).
|
|
71
|
+
*/
|
|
72
|
+
export type JobLevelOutage = 'database_unavailable' | 'pixiv_auth_failure' | 'delivery_unavailable' | 'network_outage';
|
|
73
|
+
/**
|
|
74
|
+
* What one failed candidate ATTEMPT means. `candidate` => the scan advances.
|
|
75
|
+
* `job` => the scan must not be allowed to degrade the run into an empty
|
|
76
|
+
* candidate list.
|
|
77
|
+
*/
|
|
78
|
+
export type CandidateFailure = {
|
|
79
|
+
scope: 'candidate';
|
|
80
|
+
skip: CandidateSkip;
|
|
81
|
+
} | {
|
|
82
|
+
scope: 'job';
|
|
83
|
+
outage: JobLevelOutage;
|
|
84
|
+
error: string;
|
|
85
|
+
};
|
|
86
|
+
/** What happened to ONE candidate work during a target's bounded scan. */
|
|
87
|
+
export type CandidateAttempt =
|
|
88
|
+
/** The candidate produced the target's business result (or a delivery intent). */
|
|
89
|
+
{
|
|
90
|
+
kind: 'selected';
|
|
91
|
+
workId: string;
|
|
92
|
+
workType: WorkType;
|
|
93
|
+
}
|
|
94
|
+
/** The candidate was unusable; the scan continues with the next one. */
|
|
95
|
+
| {
|
|
96
|
+
kind: 'skipped';
|
|
97
|
+
skip: CandidateSkip;
|
|
98
|
+
};
|
|
99
|
+
/**
|
|
100
|
+
* Bookkeeping for a target's bounded candidate scan. Attached to the terminal
|
|
101
|
+
* target outcome so the run can state exactly one of:
|
|
102
|
+
*
|
|
103
|
+
* "submitted candidate X after skipping Y candidate(s)"
|
|
104
|
+
* "no eligible candidate found after scanning N"
|
|
105
|
+
*
|
|
106
|
+
* `bound` is the configured cap (never exceeded), `attempted` the real count.
|
|
107
|
+
*/
|
|
108
|
+
export interface CandidateScanSummary {
|
|
109
|
+
/**
|
|
110
|
+
* Cap on candidates this scan was allowed to attempt: the effective window,
|
|
111
|
+
* never above the configured `candidateScanLimit` unless the target's own
|
|
112
|
+
* `limit` is larger (a multi-work target must be able to fill its limit).
|
|
113
|
+
*/
|
|
114
|
+
bound: number;
|
|
115
|
+
/** Candidates actually attempted. Always <= bound. */
|
|
116
|
+
attempted: number;
|
|
117
|
+
/**
|
|
118
|
+
* Candidates skipped before the scan ended, in the order they were skipped —
|
|
119
|
+
* strictly candidate order whenever the scan is serial, which is always the
|
|
120
|
+
* case for a single-work cell.
|
|
121
|
+
*/
|
|
122
|
+
skipped: CandidateSkip[];
|
|
123
|
+
/** Job-level outages observed while scanning (never a candidate verdict). */
|
|
124
|
+
outages: JobLevelOutage[];
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* A scan that attempted nothing. The real pipeline always reports its own scan;
|
|
128
|
+
* this is for callers/tests that have no candidate-level information, so they
|
|
129
|
+
* cannot fabricate a verdict they did not observe.
|
|
130
|
+
*/
|
|
131
|
+
export declare function emptyCandidateScan(): CandidateScanSummary;
|
|
10
132
|
export type TargetOutcome = {
|
|
11
133
|
kind: 'submitted';
|
|
12
134
|
workId: string;
|
|
13
135
|
workType: WorkType;
|
|
14
136
|
/** Durable delivery row that recorded the downstream ACK. */
|
|
15
137
|
deliveryId?: string;
|
|
138
|
+
/** Which candidates were skipped to reach this one. */
|
|
139
|
+
scan?: CandidateScanSummary;
|
|
16
140
|
}
|
|
17
141
|
/**
|
|
18
142
|
* The work was processed locally but there is no downstream delivery target
|
|
@@ -24,6 +148,7 @@ export type TargetOutcome = {
|
|
|
24
148
|
kind: 'stored';
|
|
25
149
|
workId: string;
|
|
26
150
|
workType: WorkType;
|
|
151
|
+
scan?: CandidateScanSummary;
|
|
27
152
|
}
|
|
28
153
|
/**
|
|
29
154
|
* A delivery intent + outbox item were created durably, but the downstream
|
|
@@ -35,29 +160,97 @@ export type TargetOutcome = {
|
|
|
35
160
|
workId: string;
|
|
36
161
|
workType: WorkType;
|
|
37
162
|
deliveryId: string;
|
|
163
|
+
scan?: CandidateScanSummary;
|
|
38
164
|
}
|
|
39
|
-
/**
|
|
165
|
+
/**
|
|
166
|
+
* The bounded scan found no ELIGIBLE candidate: every attempted candidate was
|
|
167
|
+
* skipped for a candidate-level reason (duplicate / deleted / denied / ...).
|
|
168
|
+
* A clean no-op, not a failure and not a retry loop.
|
|
169
|
+
*/
|
|
40
170
|
| {
|
|
41
171
|
kind: 'no_candidate';
|
|
42
172
|
reason: string;
|
|
173
|
+
scan?: CandidateScanSummary;
|
|
43
174
|
}
|
|
44
175
|
/**
|
|
45
176
|
* Downstream proved this work was already delivered by a DIFFERENT intent
|
|
46
177
|
* (historical drift). This is a terminal business duplicate, not a new
|
|
47
|
-
* submission. Produced only by the explicit reconciliation path
|
|
178
|
+
* submission. Produced only by the explicit reconciliation path
|
|
179
|
+
* (`settleDeliveryTerminal`) or by a single-work cell RESUME whose locked work
|
|
180
|
+
* turns out to be already delivered — NEVER as the verdict of a candidate scan.
|
|
48
181
|
*/
|
|
49
182
|
| {
|
|
50
183
|
kind: 'duplicate';
|
|
51
184
|
workId: string;
|
|
52
185
|
reason: string;
|
|
186
|
+
scan?: CandidateScanSummary;
|
|
53
187
|
}
|
|
54
188
|
/** The target failed. retryable=true => a later trigger/outbox may resume. */
|
|
55
189
|
| {
|
|
56
190
|
kind: 'failed';
|
|
57
191
|
retryable: boolean;
|
|
58
192
|
error: string;
|
|
193
|
+
scan?: CandidateScanSummary;
|
|
59
194
|
};
|
|
60
195
|
/** Terminal outcomes settle the cell; others leave it recoverable. */
|
|
61
196
|
export declare function isTerminalOutcome(outcome: TargetOutcome): boolean;
|
|
197
|
+
/** True when the scan ended on a candidate it may actually submit. */
|
|
198
|
+
export declare function isSelectedAttempt(attempt: CandidateAttempt | void): attempt is {
|
|
199
|
+
kind: 'selected';
|
|
200
|
+
workId: string;
|
|
201
|
+
workType: WorkType;
|
|
202
|
+
};
|
|
203
|
+
/**
|
|
204
|
+
* True when this candidate failure means "move on to the next candidate"
|
|
205
|
+
* WITHOUT retrying this one — the cause is a fact about the work (deleted,
|
|
206
|
+
* private, wrong media/format, filtered by rules, already submitted).
|
|
207
|
+
*
|
|
208
|
+
* Transient infrastructure failures return FALSE on purpose: the existing
|
|
209
|
+
* retry/backoff semantics own those, because retrying the same work is what
|
|
210
|
+
* this repo does, and churning through a whole ranking page while the network
|
|
211
|
+
* is down would be worse than failing the target once.
|
|
212
|
+
*/
|
|
213
|
+
export declare function skipCandidateWithoutRetry(skip: CandidateSkip): boolean;
|
|
214
|
+
/**
|
|
215
|
+
* The explicit, human-readable verdict of a target. Requirement: a run must
|
|
216
|
+
* report one of "submitted candidate X after skipping Y candidates" or "no
|
|
217
|
+
* eligible candidate found after scanning N" — never a bare "completed".
|
|
218
|
+
*/
|
|
62
219
|
export declare function outcomeSummary(outcome: TargetOutcome): string;
|
|
220
|
+
/**
|
|
221
|
+
* `no eligible candidate found after scanning N` (+ what was skipped).
|
|
222
|
+
*
|
|
223
|
+
* Candidates dropped before any download attempt (already submitted / already
|
|
224
|
+
* in history) are counted separately from the ones actually attempted, because
|
|
225
|
+
* "scanned 0" with three duplicates is a very different statement from
|
|
226
|
+
* "scanned 3, all unusable" — and the operator needs to be able to tell them
|
|
227
|
+
* apart.
|
|
228
|
+
*/
|
|
229
|
+
export declare function noEligibleCandidateText(scan: CandidateScanSummary): string;
|
|
230
|
+
/**
|
|
231
|
+
* True when the scan saw a transient infrastructure failure, or ANY attempted
|
|
232
|
+
* candidate failed transiently. Such a scan says "the network/API is flaky",
|
|
233
|
+
* NOT "no eligible candidate", so the caller must fail/retry the target instead
|
|
234
|
+
* of reporting a clean empty scan.
|
|
235
|
+
*/
|
|
236
|
+
export declare function hasTransientFailure(scan: CandidateScanSummary): boolean;
|
|
237
|
+
/**
|
|
238
|
+
* Fold two scans of the same logical target into one, so a multi-page or
|
|
239
|
+
* lookback scan reports the whole picture rather than only its last page.
|
|
240
|
+
* `bound` is the sum of the windows actually offered (each page is bounded
|
|
241
|
+
* independently), which keeps `attempted <= bound` true.
|
|
242
|
+
*/
|
|
243
|
+
export declare function mergeScanSummaries(first: CandidateScanSummary | null, second: CandidateScanSummary): CandidateScanSummary;
|
|
244
|
+
/**
|
|
245
|
+
* Decide what a failed candidate ATTEMPT means.
|
|
246
|
+
*
|
|
247
|
+
* This is the boundary the previous design got wrong: a duplicate was treated
|
|
248
|
+
* as a completed job. The order below is what makes the distinction real —
|
|
249
|
+
* a dead database, a dead token, a dead delivery provider or a dead network is
|
|
250
|
+
* a JOB problem and is classified as such before any candidate-level pattern
|
|
251
|
+
* can claim it.
|
|
252
|
+
*/
|
|
253
|
+
export declare function classifyCandidateFailure(error: unknown, workId: string): CandidateFailure;
|
|
254
|
+
/** Hard job-level outage for a directly-thrown error, or null. */
|
|
255
|
+
export declare function classifyJobLevelOutage(error: unknown): JobLevelOutage | null;
|
|
63
256
|
//# sourceMappingURL=TargetOutcome.d.ts.map
|
|
@@ -1,7 +1,23 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.emptyCandidateScan = emptyCandidateScan;
|
|
3
4
|
exports.isTerminalOutcome = isTerminalOutcome;
|
|
5
|
+
exports.isSelectedAttempt = isSelectedAttempt;
|
|
6
|
+
exports.skipCandidateWithoutRetry = skipCandidateWithoutRetry;
|
|
4
7
|
exports.outcomeSummary = outcomeSummary;
|
|
8
|
+
exports.noEligibleCandidateText = noEligibleCandidateText;
|
|
9
|
+
exports.hasTransientFailure = hasTransientFailure;
|
|
10
|
+
exports.mergeScanSummaries = mergeScanSummaries;
|
|
11
|
+
exports.classifyCandidateFailure = classifyCandidateFailure;
|
|
12
|
+
exports.classifyJobLevelOutage = classifyJobLevelOutage;
|
|
13
|
+
/**
|
|
14
|
+
* A scan that attempted nothing. The real pipeline always reports its own scan;
|
|
15
|
+
* this is for callers/tests that have no candidate-level information, so they
|
|
16
|
+
* cannot fabricate a verdict they did not observe.
|
|
17
|
+
*/
|
|
18
|
+
function emptyCandidateScan() {
|
|
19
|
+
return { bound: 0, attempted: 0, skipped: [], outages: [] };
|
|
20
|
+
}
|
|
5
21
|
/** Terminal outcomes settle the cell; others leave it recoverable. */
|
|
6
22
|
function isTerminalOutcome(outcome) {
|
|
7
23
|
return (outcome.kind === 'submitted' ||
|
|
@@ -10,19 +26,202 @@ function isTerminalOutcome(outcome) {
|
|
|
10
26
|
outcome.kind === 'duplicate' ||
|
|
11
27
|
(outcome.kind === 'failed' && !outcome.retryable));
|
|
12
28
|
}
|
|
29
|
+
/** True when the scan ended on a candidate it may actually submit. */
|
|
30
|
+
function isSelectedAttempt(attempt) {
|
|
31
|
+
return attempt?.kind === 'selected';
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* True when this candidate failure means "move on to the next candidate"
|
|
35
|
+
* WITHOUT retrying this one — the cause is a fact about the work (deleted,
|
|
36
|
+
* private, wrong media/format, filtered by rules, already submitted).
|
|
37
|
+
*
|
|
38
|
+
* Transient infrastructure failures return FALSE on purpose: the existing
|
|
39
|
+
* retry/backoff semantics own those, because retrying the same work is what
|
|
40
|
+
* this repo does, and churning through a whole ranking page while the network
|
|
41
|
+
* is down would be worse than failing the target once.
|
|
42
|
+
*/
|
|
43
|
+
function skipCandidateWithoutRetry(skip) {
|
|
44
|
+
return skip.retryable !== true;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* The explicit, human-readable verdict of a target. Requirement: a run must
|
|
48
|
+
* report one of "submitted candidate X after skipping Y candidates" or "no
|
|
49
|
+
* eligible candidate found after scanning N" — never a bare "completed".
|
|
50
|
+
*/
|
|
13
51
|
function outcomeSummary(outcome) {
|
|
14
52
|
switch (outcome.kind) {
|
|
15
53
|
case 'submitted':
|
|
16
54
|
case 'stored':
|
|
17
|
-
return outcome.
|
|
55
|
+
return outcome.scan
|
|
56
|
+
? `submitted candidate ${outcome.workId} (${outcome.workType}) after skipping ` +
|
|
57
|
+
`${outcome.scan.skipped.length} candidate(s)${skipDetail(outcome.scan)}`
|
|
58
|
+
: `${outcome.kind} ${outcome.workId}`;
|
|
18
59
|
case 'delivery_pending':
|
|
19
|
-
return
|
|
60
|
+
return outcome.scan
|
|
61
|
+
? `submitted candidate ${outcome.workId} (${outcome.workType}) for review after skipping ` +
|
|
62
|
+
`${outcome.scan.skipped.length} candidate(s)${skipDetail(outcome.scan)}`
|
|
63
|
+
: `delivery_pending ${outcome.workId}`;
|
|
20
64
|
case 'no_candidate':
|
|
21
|
-
return
|
|
65
|
+
return outcome.scan
|
|
66
|
+
? noEligibleCandidateText(outcome.scan)
|
|
67
|
+
: `no_candidate: ${outcome.reason}`;
|
|
22
68
|
case 'duplicate':
|
|
23
69
|
return `duplicate: ${outcome.reason}`;
|
|
24
70
|
case 'failed':
|
|
25
71
|
return `failed(${outcome.retryable ? 'retryable' : 'permanent'}): ${outcome.error}`;
|
|
26
72
|
}
|
|
27
73
|
}
|
|
74
|
+
/**
|
|
75
|
+
* `no eligible candidate found after scanning N` (+ what was skipped).
|
|
76
|
+
*
|
|
77
|
+
* Candidates dropped before any download attempt (already submitted / already
|
|
78
|
+
* in history) are counted separately from the ones actually attempted, because
|
|
79
|
+
* "scanned 0" with three duplicates is a very different statement from
|
|
80
|
+
* "scanned 3, all unusable" — and the operator needs to be able to tell them
|
|
81
|
+
* apart.
|
|
82
|
+
*/
|
|
83
|
+
function noEligibleCandidateText(scan) {
|
|
84
|
+
const scanned = scan.attempted;
|
|
85
|
+
if (scan.skipped.length === 0) {
|
|
86
|
+
return `no eligible candidate found after scanning ${scanned}`;
|
|
87
|
+
}
|
|
88
|
+
const codes = [...new Set(scan.skipped.map((s) => s.code))].join(', ');
|
|
89
|
+
const prefiltered = Math.max(0, scan.skipped.length - scanned);
|
|
90
|
+
const detail = prefiltered > 0
|
|
91
|
+
? ` (bound ${scan.bound}); all ${scan.skipped.length} candidate(s) unusable ` +
|
|
92
|
+
`[${prefiltered} filtered before download, ${scanned} attempted]: ${codes}`
|
|
93
|
+
: ` (bound ${scan.bound}); all ${scan.skipped.length} attempted candidate(s) skipped: ${codes}`;
|
|
94
|
+
return `no eligible candidate found after scanning ${scanned}${detail}${skipDetail(scan)}`;
|
|
95
|
+
}
|
|
96
|
+
function skipDetail(scan) {
|
|
97
|
+
if (scan.skipped.length === 0)
|
|
98
|
+
return '';
|
|
99
|
+
const sample = scan.skipped
|
|
100
|
+
.slice(0, 3)
|
|
101
|
+
.map((s) => `${s.code}(${s.workId})`)
|
|
102
|
+
.join(', ');
|
|
103
|
+
return ` [${sample}${scan.skipped.length > 3 ? `, +${scan.skipped.length - 3} more` : ''}]`;
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* True when the scan saw a transient infrastructure failure, or ANY attempted
|
|
107
|
+
* candidate failed transiently. Such a scan says "the network/API is flaky",
|
|
108
|
+
* NOT "no eligible candidate", so the caller must fail/retry the target instead
|
|
109
|
+
* of reporting a clean empty scan.
|
|
110
|
+
*/
|
|
111
|
+
function hasTransientFailure(scan) {
|
|
112
|
+
if (scan.outages.length > 0)
|
|
113
|
+
return true;
|
|
114
|
+
return scan.skipped.some((skip) => skip.retryable === true);
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Fold two scans of the same logical target into one, so a multi-page or
|
|
118
|
+
* lookback scan reports the whole picture rather than only its last page.
|
|
119
|
+
* `bound` is the sum of the windows actually offered (each page is bounded
|
|
120
|
+
* independently), which keeps `attempted <= bound` true.
|
|
121
|
+
*/
|
|
122
|
+
function mergeScanSummaries(first, second) {
|
|
123
|
+
if (!first)
|
|
124
|
+
return second;
|
|
125
|
+
return {
|
|
126
|
+
bound: first.bound + second.bound,
|
|
127
|
+
attempted: first.attempted + second.attempted,
|
|
128
|
+
skipped: [...first.skipped, ...second.skipped],
|
|
129
|
+
outages: [...new Set([...first.outages, ...second.outages])],
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
const SQLITE_OUTAGE = /sqlite|database (?:is )?locked|unable to open database|no such table|disk i\/o error/i;
|
|
133
|
+
/** A delivery-provider OUTAGE — not a per-message rejection such as a too-long caption. */
|
|
134
|
+
const DELIVERY_OUTAGE = /(?:telegram|delivery (?:target|provider))[^.]{0,80}\b(?:unavailable|unreachable|down|not configured|timed? ?out|econn\w*|etimedout|enotfound|401|403|50[234])\b|api\.telegram\.org[^.]*\b(?:econn\w*|etimedout|enotfound|50[234])\b/i;
|
|
135
|
+
const AUTH_OUTAGE = /\b(401|unauthorized|invalid_grant|invalid refresh token|authentication failed)\b/i;
|
|
136
|
+
const NETWORK_OUTAGE = /econnrefused|econnreset|enotfound|etimedout|ehostunreach|enetunreach|socket hang up|network is unreachable|getaddrinfo/i;
|
|
137
|
+
const RATE_LIMIT = /\b429\b|rate limit/i;
|
|
138
|
+
function messageOf(error) {
|
|
139
|
+
if (error instanceof Error) {
|
|
140
|
+
return `${error.name}: ${error.message}`;
|
|
141
|
+
}
|
|
142
|
+
return String(error);
|
|
143
|
+
}
|
|
144
|
+
function statusOf(error) {
|
|
145
|
+
const status = error?.status ??
|
|
146
|
+
error?.statusCode;
|
|
147
|
+
return typeof status === 'number' ? status : undefined;
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* Decide what a failed candidate ATTEMPT means.
|
|
151
|
+
*
|
|
152
|
+
* This is the boundary the previous design got wrong: a duplicate was treated
|
|
153
|
+
* as a completed job. The order below is what makes the distinction real —
|
|
154
|
+
* a dead database, a dead token, a dead delivery provider or a dead network is
|
|
155
|
+
* a JOB problem and is classified as such before any candidate-level pattern
|
|
156
|
+
* can claim it.
|
|
157
|
+
*/
|
|
158
|
+
function classifyCandidateFailure(error, workId) {
|
|
159
|
+
const message = messageOf(error);
|
|
160
|
+
const status = statusOf(error);
|
|
161
|
+
const name = error instanceof Error ? error.name : '';
|
|
162
|
+
const code = error?.code;
|
|
163
|
+
const codeText = typeof code === 'string' ? code : '';
|
|
164
|
+
// --- Job-level first: these must never shrink to "empty candidate list" ---
|
|
165
|
+
if (name === 'DatabaseError' || codeText.startsWith('SQLITE_') || SQLITE_OUTAGE.test(message)) {
|
|
166
|
+
return { scope: 'job', outage: 'database_unavailable', error: message };
|
|
167
|
+
}
|
|
168
|
+
if (name === 'AuthenticationError' || status === 401 || AUTH_OUTAGE.test(message)) {
|
|
169
|
+
return { scope: 'job', outage: 'pixiv_auth_failure', error: message };
|
|
170
|
+
}
|
|
171
|
+
// The delivery provider being unreachable says nothing about the candidate:
|
|
172
|
+
// every candidate would "fail" identically, so this must never be counted as
|
|
173
|
+
// an empty candidate list.
|
|
174
|
+
if (DELIVERY_OUTAGE.test(message) && !/already delivered|already published|duplicate/i.test(message)) {
|
|
175
|
+
return { scope: 'job', outage: 'delivery_unavailable', error: message };
|
|
176
|
+
}
|
|
177
|
+
if (status === 502 || status === 503 || status === 504) {
|
|
178
|
+
// The remote provider itself is down, not this work.
|
|
179
|
+
return { scope: 'job', outage: 'network_outage', error: message };
|
|
180
|
+
}
|
|
181
|
+
if (NETWORK_OUTAGE.test(message)) {
|
|
182
|
+
return {
|
|
183
|
+
scope: 'candidate',
|
|
184
|
+
skip: { code: 'unavailable', workId, reason: message, retryable: true },
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
// --- Candidate-level: this ONE work is unusable, try the next one ---------
|
|
188
|
+
if (/already delivered|already processed|already published|idempotent_replay/i.test(message)) {
|
|
189
|
+
return {
|
|
190
|
+
scope: 'candidate',
|
|
191
|
+
skip: { code: 'duplicate', workId, reason: message },
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
if (name === 'PixivNotFoundError' || status === 404 || /\b404\b|not found|deleted/i.test(message)) {
|
|
195
|
+
return { scope: 'candidate', skip: { code: 'deleted', workId, reason: message } };
|
|
196
|
+
}
|
|
197
|
+
if (name === 'PixivRateLimitError' || RATE_LIMIT.test(message)) {
|
|
198
|
+
return {
|
|
199
|
+
scope: 'candidate',
|
|
200
|
+
skip: { code: 'unavailable', workId, reason: message, retryable: true },
|
|
201
|
+
};
|
|
202
|
+
}
|
|
203
|
+
if (status === 403 || /forbidden|private|access denied|permission/i.test(message)) {
|
|
204
|
+
return { scope: 'candidate', skip: { code: 'access_denied', workId, reason: message } };
|
|
205
|
+
}
|
|
206
|
+
if (/ugoira|unsupported (?:media|type|format)|cannot (?:process|handle)/i.test(message)) {
|
|
207
|
+
return { scope: 'candidate', skip: { code: 'unsupported_media', workId, reason: message } };
|
|
208
|
+
}
|
|
209
|
+
if (/language filter|filtered out|excluded (?:by|from)/i.test(message)) {
|
|
210
|
+
return { scope: 'candidate', skip: { code: 'filtered', workId, reason: message } };
|
|
211
|
+
}
|
|
212
|
+
if (/invalid (?:metadata|id|illustId|novelId)|missing (?:metadata|page|image)|no files produced/i.test(message)) {
|
|
213
|
+
return { scope: 'candidate', skip: { code: 'invalid_metadata', workId, reason: message } };
|
|
214
|
+
}
|
|
215
|
+
// Unclassifiable: treat as a transient candidate problem (retryable) so an
|
|
216
|
+
// all-unknown scan fails the job rather than silently reporting an empty scan.
|
|
217
|
+
return {
|
|
218
|
+
scope: 'candidate',
|
|
219
|
+
skip: { code: 'unavailable', workId, reason: message, retryable: true },
|
|
220
|
+
};
|
|
221
|
+
}
|
|
222
|
+
/** Hard job-level outage for a directly-thrown error, or null. */
|
|
223
|
+
function classifyJobLevelOutage(error) {
|
|
224
|
+
const failure = classifyCandidateFailure(error, '');
|
|
225
|
+
return failure.scope === 'job' ? failure.outage : null;
|
|
226
|
+
}
|
|
28
227
|
//# sourceMappingURL=TargetOutcome.js.map
|
|
@@ -60,6 +60,18 @@ export declare class DeliveryRepository extends BaseRepository {
|
|
|
60
60
|
isDelivered(deliveryTarget: string, workType: string, pixivId: string): boolean;
|
|
61
61
|
/** Batch form for candidate-pipeline pre-lock dedupe. */
|
|
62
62
|
deliveredIds(deliveryTarget: string, workType: string, pixivIds: string[]): Set<string>;
|
|
63
|
+
/**
|
|
64
|
+
* Batch pre-lock dedupe for CANDIDATE SELECTION: works already delivered, or
|
|
65
|
+
* whose review submission is still PENDING.
|
|
66
|
+
*
|
|
67
|
+
* Deliberately broader than `deliveredIds`: a pending intent is a work already
|
|
68
|
+
* submitted for human review that has not been answered yet. Re-selecting it
|
|
69
|
+
* would submit the same work a second time, which is the user-visible defect
|
|
70
|
+
* this scan exists to prevent. Within-slot RESUME keeps using `isDelivered`:
|
|
71
|
+
* a cell resuming its OWN pending work is continuing it, not duplicating it.
|
|
72
|
+
*/
|
|
73
|
+
submittedIds(deliveryTarget: string, workType: string, pixivIds: string[]): Set<string>;
|
|
74
|
+
private idsByStatus;
|
|
63
75
|
recordAck(id: string, ack: {
|
|
64
76
|
status: DeliveryStatus;
|
|
65
77
|
remoteId?: string;
|
|
@@ -71,16 +71,33 @@ class DeliveryRepository extends BaseRepository_1.BaseRepository {
|
|
|
71
71
|
}
|
|
72
72
|
/** Batch form for candidate-pipeline pre-lock dedupe. */
|
|
73
73
|
deliveredIds(deliveryTarget, workType, pixivIds) {
|
|
74
|
+
return this.idsByStatus(deliveryTarget, workType, pixivIds, ['delivered', 'duplicate']);
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Batch pre-lock dedupe for CANDIDATE SELECTION: works already delivered, or
|
|
78
|
+
* whose review submission is still PENDING.
|
|
79
|
+
*
|
|
80
|
+
* Deliberately broader than `deliveredIds`: a pending intent is a work already
|
|
81
|
+
* submitted for human review that has not been answered yet. Re-selecting it
|
|
82
|
+
* would submit the same work a second time, which is the user-visible defect
|
|
83
|
+
* this scan exists to prevent. Within-slot RESUME keeps using `isDelivered`:
|
|
84
|
+
* a cell resuming its OWN pending work is continuing it, not duplicating it.
|
|
85
|
+
*/
|
|
86
|
+
submittedIds(deliveryTarget, workType, pixivIds) {
|
|
87
|
+
return this.idsByStatus(deliveryTarget, workType, pixivIds, ['pending', 'delivered', 'duplicate']);
|
|
88
|
+
}
|
|
89
|
+
idsByStatus(deliveryTarget, workType, pixivIds, statuses) {
|
|
74
90
|
const out = new Set();
|
|
75
91
|
if (pixivIds.length === 0)
|
|
76
92
|
return out;
|
|
93
|
+
const statusList = statuses.map((status) => `'${status}'`).join(',');
|
|
77
94
|
const CHUNK = 400;
|
|
78
95
|
for (let i = 0; i < pixivIds.length; i += CHUNK) {
|
|
79
96
|
const slice = pixivIds.slice(i, i + CHUNK);
|
|
80
97
|
const placeholders = slice.map(() => '?').join(',');
|
|
81
98
|
const rows = this.db
|
|
82
99
|
.prepare(`SELECT DISTINCT pixiv_id FROM deliveries
|
|
83
|
-
WHERE delivery_target = ? AND work_type = ? AND status IN (
|
|
100
|
+
WHERE delivery_target = ? AND work_type = ? AND status IN (${statusList})
|
|
84
101
|
AND pixiv_id IN (${placeholders})`)
|
|
85
102
|
.all(deliveryTarget, workType, ...slice);
|
|
86
103
|
for (const r of rows)
|
package/dist/version.js
CHANGED
|
@@ -2,5 +2,5 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.BUILD = void 0;
|
|
4
4
|
// GENERATED by scripts/write-version.js — do not edit manually.
|
|
5
|
-
exports.BUILD = { version: '2.19.
|
|
5
|
+
exports.BUILD = { version: '2.19.5', commit: '945772649d9f' };
|
|
6
6
|
//# sourceMappingURL=version.js.map
|
package/dist/webui/package.json
CHANGED
package/package.json
CHANGED