pixivflow 2.19.4 → 2.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/SchedulerCommand.js +36 -0
- package/dist/commands/SchedulerRunOnceCommand.d.ts +2 -3
- package/dist/commands/SchedulerRunOnceCommand.js +2 -3
- package/dist/commands/scheduler-runtime.js +17 -1
- package/dist/config/defaults.d.ts +1 -0
- package/dist/config/defaults.js +4 -0
- package/dist/config/environment.js +19 -0
- package/dist/config/types.d.ts +28 -0
- package/dist/config/validation.js +25 -0
- package/dist/delivery/DeliveryService.d.ts +30 -1
- package/dist/delivery/DeliveryService.js +13 -2
- package/dist/delivery/HttpMultipartDelivery.js +29 -8
- package/dist/delivery/OutboxWorker.d.ts +2 -0
- package/dist/delivery/OutboxWorker.js +3 -0
- package/dist/delivery/types.d.ts +18 -0
- package/dist/download/DownloadManager.js +12 -1
- package/dist/download/handlers/IllustrationTargetHandler.d.ts +42 -2
- package/dist/download/handlers/IllustrationTargetHandler.js +240 -50
- package/dist/download/handlers/NovelTargetHandler.d.ts +47 -0
- package/dist/download/handlers/NovelTargetHandler.js +213 -27
- package/dist/download/handlers/deliveryContext.d.ts +16 -0
- package/dist/download/handlers/deliveryContext.js +32 -0
- package/dist/download/pipeline/DownloadPipeline.d.ts +22 -3
- package/dist/download/pipeline/DownloadPipeline.js +109 -92
- package/dist/download/plan/DownloadPlanner.d.ts +47 -1
- package/dist/download/plan/DownloadPlanner.js +105 -15
- package/dist/notification/NotificationPolicy.d.ts +12 -0
- package/dist/notification/NotificationPolicy.js +86 -0
- package/dist/package.json +1 -1
- package/dist/scheduler/MultiScheduleManager.js +2 -0
- package/dist/scheduler/ScheduleTriggerServer.d.ts +58 -1
- package/dist/scheduler/ScheduleTriggerServer.js +207 -40
- package/dist/scheduler/SlotCoordinator.d.ts +80 -2
- package/dist/scheduler/SlotCoordinator.js +145 -3
- package/dist/scheduler/TargetOutcome.d.ts +195 -2
- package/dist/scheduler/TargetOutcome.js +202 -3
- package/dist/storage/DatabaseMigration.js +11 -0
- package/dist/storage/repositories/DeliveryRepository.d.ts +12 -0
- package/dist/storage/repositories/DeliveryRepository.js +18 -1
- package/dist/storage/repositories/SlotRepository.d.ts +8 -0
- package/dist/storage/repositories/SlotRepository.js +6 -2
- package/dist/version.js +1 -1
- package/dist/webui/package.json +1 -1
- package/package.json +1 -1
|
@@ -25,6 +25,31 @@ function deliveryTargetOf(target) {
|
|
|
25
25
|
const name = target.delivery?.target;
|
|
26
26
|
return typeof name === 'string' && name.trim() ? name.trim() : null;
|
|
27
27
|
}
|
|
28
|
+
/**
|
|
29
|
+
* SQLite `CURRENT_TIMESTAMP` is UTC "YYYY-MM-DD HH:MM:SS". It carries no zone
|
|
30
|
+
* marker, and `Date.parse` reads that shape as LOCAL time — so the zone is
|
|
31
|
+
* added explicitly instead of being trusted to the engine.
|
|
32
|
+
*/
|
|
33
|
+
function sqliteUtcMs(value) {
|
|
34
|
+
if (!value)
|
|
35
|
+
return undefined;
|
|
36
|
+
const normalized = /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/.test(value)
|
|
37
|
+
? `${value.replace(' ', 'T')}Z`
|
|
38
|
+
: value;
|
|
39
|
+
const ms = Date.parse(normalized);
|
|
40
|
+
return Number.isNaN(ms) ? undefined : ms;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Epoch ms -> ISO 8601 UTC, or undefined when it is not a usable instant.
|
|
44
|
+
* `toISOString` throws on an out-of-range Date, and a throw here would abort the
|
|
45
|
+
* terminal rollup — a logging field must never be able to break dispatch.
|
|
46
|
+
*/
|
|
47
|
+
function isoUtcOrUndefined(epochMs) {
|
|
48
|
+
if (typeof epochMs !== 'number' || !Number.isFinite(epochMs))
|
|
49
|
+
return undefined;
|
|
50
|
+
const date = new Date(epochMs);
|
|
51
|
+
return Number.isNaN(date.getTime()) ? undefined : date.toISOString();
|
|
52
|
+
}
|
|
28
53
|
/**
|
|
29
54
|
* Owns the Schedule Slot ledger for one scheduler run. A Slot is one durable
|
|
30
55
|
* execution occurrence of a Schedule (NOT a morning/evening row). It ensures
|
|
@@ -102,6 +127,8 @@ class SlotCoordinator {
|
|
|
102
127
|
triggerSource: slot.triggerSource,
|
|
103
128
|
slotDate: slot.slotDate,
|
|
104
129
|
slotName: slot.slotName,
|
|
130
|
+
manualRequestId: slot.manualRequestId ?? null,
|
|
131
|
+
correlationId: slot.correlationId ?? null,
|
|
105
132
|
});
|
|
106
133
|
if (created) {
|
|
107
134
|
// Freeze membership: materialize one cell per target id from the snapshot.
|
|
@@ -347,25 +374,139 @@ class SlotCoordinator {
|
|
|
347
374
|
}
|
|
348
375
|
const status = this.database.slots.deriveSlotStatus(slot.slotId);
|
|
349
376
|
this.database.slots.markSlotStatus(slot.slotId, status);
|
|
350
|
-
this.persistExecutionSummary(slot.slotId, status);
|
|
351
377
|
const cells = this.database.slots.getCells(slot.slotId).map((c) => ({
|
|
352
378
|
targetId: c.targetId,
|
|
353
379
|
status: c.status,
|
|
354
380
|
workId: c.workId,
|
|
355
381
|
error: c.lastError,
|
|
356
382
|
}));
|
|
383
|
+
// Rolled up AFTER the slot row carries its terminal status/completed_at, so
|
|
384
|
+
// the outcome reports the durable timestamps rather than a fresh clock read.
|
|
385
|
+
this.persistExecutionSummary(slot.slotId, status, this.scheduleOutcome(slot, status, targets));
|
|
357
386
|
const icon = (s) => s === 'submitted' ? '✅' : s === 'no_candidate' ? '⚠️ no_candidate' : '❌ failed';
|
|
358
387
|
logger_1.logger.info(`Slot ${slot.slotId} ${status}\n` +
|
|
359
388
|
cells.map((c) => ` ${c.targetId.padEnd(28)} ${icon(c.status)} ${c.workId ? '#' + c.workId : ''} ${c.error ? '(' + c.error + ')' : ''}`).join('\n'), { slot: slot.slotId, status });
|
|
360
389
|
return { scheduleId: schedule.id, slotId: slot.slotId, status, alreadyCompleted: false, cells };
|
|
361
390
|
}
|
|
391
|
+
/**
|
|
392
|
+
* Project the terminal slot row into the one structured outcome record. Every
|
|
393
|
+
* field comes from durable state — the slot row, its cells, and the delivery
|
|
394
|
+
* ledger. Nothing is inferred from error-message text.
|
|
395
|
+
*/
|
|
396
|
+
scheduleOutcome(slot, status, targets) {
|
|
397
|
+
const slotRec = this.database.slots.getSlot(slot.slotId);
|
|
398
|
+
const cellRows = this.database.slots.getCells(slot.slotId);
|
|
399
|
+
// Only targets the caller still knows about can name a delivery channel; a
|
|
400
|
+
// cell with no known channel has no delivery fact to consult.
|
|
401
|
+
const deliveryTargetByTargetId = new Map();
|
|
402
|
+
for (const target of targets) {
|
|
403
|
+
const deliveryTarget = deliveryTargetOf(target);
|
|
404
|
+
if (target.id && deliveryTarget)
|
|
405
|
+
deliveryTargetByTargetId.set(target.id, deliveryTarget);
|
|
406
|
+
}
|
|
407
|
+
let submitted = 0;
|
|
408
|
+
let no_match = 0;
|
|
409
|
+
let duplicate = 0;
|
|
410
|
+
let executor_failed = 0;
|
|
411
|
+
let delivery_failed = 0;
|
|
412
|
+
const targetsDetail = cellRows.map((cell) => {
|
|
413
|
+
const classified = this.classifyCell(cell.status, slot.slotId, cell.targetId, deliveryTargetByTargetId);
|
|
414
|
+
if (classified === 'submitted')
|
|
415
|
+
submitted += 1;
|
|
416
|
+
else if (classified === 'no_match')
|
|
417
|
+
no_match += 1;
|
|
418
|
+
else if (classified === 'duplicate')
|
|
419
|
+
duplicate += 1;
|
|
420
|
+
else if (classified === 'delivery_failed')
|
|
421
|
+
delivery_failed += 1;
|
|
422
|
+
else if (classified === 'executor_failed')
|
|
423
|
+
executor_failed += 1;
|
|
424
|
+
return {
|
|
425
|
+
target_id: cell.targetId,
|
|
426
|
+
status: cell.status,
|
|
427
|
+
work_id: cell.workId,
|
|
428
|
+
error: cell.lastError,
|
|
429
|
+
};
|
|
430
|
+
});
|
|
431
|
+
// A fully-submitted slot has no non-submitted cell at all, and reporting
|
|
432
|
+
// `all_duplicates` there would be a lie — so the count must be non-zero.
|
|
433
|
+
const nonSubmitted = cellRows.length - submitted;
|
|
434
|
+
const all_duplicates = nonSubmitted > 0 && duplicate === nonSubmitted;
|
|
435
|
+
const startedMs = sqliteUtcMs(slotRec?.startedAt);
|
|
436
|
+
const completedMs = sqliteUtcMs(slotRec?.completedAt);
|
|
437
|
+
const duration_ms = startedMs !== undefined && completedMs !== undefined
|
|
438
|
+
? Math.max(0, completedMs - startedMs)
|
|
439
|
+
: undefined;
|
|
440
|
+
const occurrenceAt = slotRec?.occurrenceAt ?? slot.occurrenceAt;
|
|
441
|
+
return {
|
|
442
|
+
event: 'schedule.outcome',
|
|
443
|
+
schedule_id: slotRec?.scheduleId ?? slot.scheduleId,
|
|
444
|
+
slot_id: slot.slotId,
|
|
445
|
+
occurrence_at: isoUtcOrUndefined(occurrenceAt),
|
|
446
|
+
occurrence_date: slotRec?.occurrenceDate ?? slot.occurrenceDate,
|
|
447
|
+
status: status,
|
|
448
|
+
duration_ms,
|
|
449
|
+
cells: {
|
|
450
|
+
total: cellRows.length,
|
|
451
|
+
submitted,
|
|
452
|
+
no_match,
|
|
453
|
+
duplicate,
|
|
454
|
+
all_duplicates,
|
|
455
|
+
executor_failed,
|
|
456
|
+
delivery_failed,
|
|
457
|
+
targets: targetsDetail,
|
|
458
|
+
},
|
|
459
|
+
};
|
|
460
|
+
}
|
|
461
|
+
/**
|
|
462
|
+
* One cell -> one business category. `delivery_failed` is derived from the
|
|
463
|
+
* durable delivery ledger through the SAME port the FSM already uses, never
|
|
464
|
+
* from an error string, and it is decided BEFORE `executor_failed` because a
|
|
465
|
+
* terminally lost delivery also leaves the cell `failed`: one cause, one count.
|
|
466
|
+
*/
|
|
467
|
+
classifyCell(cellStatus, slotId, targetId, deliveryTargetByTargetId) {
|
|
468
|
+
if (cellStatus === 'submitted')
|
|
469
|
+
return 'submitted';
|
|
470
|
+
if (cellStatus === 'no_candidate')
|
|
471
|
+
return 'no_match';
|
|
472
|
+
if (cellStatus === 'duplicate')
|
|
473
|
+
return 'duplicate';
|
|
474
|
+
const deliveryTarget = deliveryTargetByTargetId.get(targetId);
|
|
475
|
+
if (deliveryTarget && this.delivery) {
|
|
476
|
+
try {
|
|
477
|
+
const state = this.delivery.stateFor({ deliveryTarget, slotId, targetId });
|
|
478
|
+
if (state.kind === 'lost')
|
|
479
|
+
return 'delivery_failed';
|
|
480
|
+
}
|
|
481
|
+
catch (error) {
|
|
482
|
+
// Observability must never break the rollup.
|
|
483
|
+
logger_1.logger.debug('Delivery state unavailable for outcome rollup', {
|
|
484
|
+
slot: slotId,
|
|
485
|
+
target: targetId,
|
|
486
|
+
error: error.message,
|
|
487
|
+
});
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
if (cellStatus === 'failed')
|
|
491
|
+
return 'executor_failed';
|
|
492
|
+
// Non-terminal (delivery_pending / artifact_ready): such a slot is not
|
|
493
|
+
// terminal either, so no outcome record is written for it at all.
|
|
494
|
+
return 'other';
|
|
495
|
+
}
|
|
362
496
|
/**
|
|
363
497
|
* Persist a one-row incident/execution summary into delivery_events
|
|
364
498
|
* (event='execution.summary') at the terminal rollup only. Event-row storage
|
|
365
499
|
* reuses the audit table + `runs show` read path instead of inventing a
|
|
366
500
|
* summary table or overloading execution_log's illustration/novel typing.
|
|
501
|
+
*
|
|
502
|
+
* The `schedule.outcome` line is emitted HERE, immediately after that durable
|
|
503
|
+
* write and behind the SAME `hasExecutionSummary` dedupe, because the summary
|
|
504
|
+
* row is the occurrence-level terminal identity that already exists. Recovery
|
|
505
|
+
* that re-rolls the same terminal slot therefore logs nothing a second time,
|
|
506
|
+
* with no shadow ledger and no second event name. The identical object goes to
|
|
507
|
+
* both sinks, so the line and the row cannot disagree.
|
|
367
508
|
*/
|
|
368
|
-
persistExecutionSummary(slotId, status) {
|
|
509
|
+
persistExecutionSummary(slotId, status, outcome) {
|
|
369
510
|
if (status !== 'success' && status !== 'partial' && status !== 'failed')
|
|
370
511
|
return;
|
|
371
512
|
try {
|
|
@@ -377,8 +518,9 @@ class SlotCoordinator {
|
|
|
377
518
|
slotId,
|
|
378
519
|
event: 'execution.summary',
|
|
379
520
|
countsAsAttempt: 0,
|
|
380
|
-
detail: { summary },
|
|
521
|
+
detail: { summary, outcome },
|
|
381
522
|
});
|
|
523
|
+
logger_1.logger.info('Schedule occurrence reached a terminal outcome', outcome);
|
|
382
524
|
}
|
|
383
525
|
catch (error) {
|
|
384
526
|
logger_1.logger.debug('Failed to persist execution summary', { slot: slotId, error: error.message });
|
|
@@ -5,14 +5,138 @@
|
|
|
5
5
|
* submitted". Undefined / a 2xx HTTP response / a caught-and-swallowed error
|
|
6
6
|
* must NEVER be inferred as a successful submission. Every target produces one
|
|
7
7
|
* of these explicit outcomes; the Slot ledger maps it to a cell transition.
|
|
8
|
+
*
|
|
9
|
+
* The vocabulary has TWO levels, and conflating them is what let a duplicate
|
|
10
|
+
* candidate end a scheduled slot as "successful":
|
|
11
|
+
*
|
|
12
|
+
* - CANDIDATE level (`CandidateSkip`): this ONE candidate work was unusable —
|
|
13
|
+
* already delivered, deleted, private, wrong media, wrong language. A
|
|
14
|
+
* candidate problem is never a job verdict: the scan advances to the next
|
|
15
|
+
* candidate.
|
|
16
|
+
* - TARGET level (`TargetOutcome`, below): the verdict for the whole logical
|
|
17
|
+
* item after its bounded candidate scan, including the scan bookkeeping so
|
|
18
|
+
* "completed with nothing done" is impossible to report silently.
|
|
8
19
|
*/
|
|
9
20
|
export type WorkType = 'illustration' | 'novel';
|
|
21
|
+
/**
|
|
22
|
+
* Why ONE candidate work was skipped without producing a business result.
|
|
23
|
+
*
|
|
24
|
+
* Codes are deliberately outcome-shaped, not error-shaped: they say what the
|
|
25
|
+
* run should DO (try the next candidate), not which exception was raised.
|
|
26
|
+
*/
|
|
27
|
+
export type CandidateSkipCode =
|
|
28
|
+
/** Already delivered / already pending review for this target, or a lost
|
|
29
|
+
* idempotency race against a concurrent worker. Try the next candidate. */
|
|
30
|
+
'duplicate'
|
|
31
|
+
/** Gone: 404, deleted work, removed account. Permanent for this candidate. */
|
|
32
|
+
| 'deleted'
|
|
33
|
+
/** Private / R-18 without permission / 403 on THIS work. */
|
|
34
|
+
| 'access_denied'
|
|
35
|
+
/** Ugoira or novel form this build cannot process. */
|
|
36
|
+
| 'unsupported_media'
|
|
37
|
+
/** Missing or malformed metadata: no id, no pages, no image urls. */
|
|
38
|
+
| 'invalid_metadata'
|
|
39
|
+
/**
|
|
40
|
+
* The attempt failed for a reason that is NOT a fact about the work: timeout,
|
|
41
|
+
* connection reset, rate limit, 5xx, or an unclassifiable error. `retryable`
|
|
42
|
+
* is set, which means the candidate is handed back to the existing
|
|
43
|
+
* retry/backoff instead of being silently skipped — and a scan in which EVERY
|
|
44
|
+
* attempted candidate was transient is a JOB failure, not an empty candidate
|
|
45
|
+
* list.
|
|
46
|
+
*/
|
|
47
|
+
| 'unavailable'
|
|
48
|
+
/**
|
|
49
|
+
* Not usable for THIS run by the run's own rules: full-text language filter,
|
|
50
|
+
* over maxPageCount, AI-metadata check, or a candidate the downloader
|
|
51
|
+
* deliberately declined. Move on to the next candidate.
|
|
52
|
+
*/
|
|
53
|
+
| 'filtered';
|
|
54
|
+
export interface CandidateSkip {
|
|
55
|
+
code: CandidateSkipCode;
|
|
56
|
+
workId: string;
|
|
57
|
+
reason: string;
|
|
58
|
+
/**
|
|
59
|
+
* True when the cause is transient infrastructure rather than this work.
|
|
60
|
+
* One such candidate still means "try the next"; EVERY attempted candidate
|
|
61
|
+
* being retryable means the infrastructure — not the candidate list — is the
|
|
62
|
+
* problem, and the target must fail/retry instead of reporting "no eligible
|
|
63
|
+
* candidate".
|
|
64
|
+
*/
|
|
65
|
+
retryable?: boolean;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Infrastructure failure scoped to the WHOLE job, never to one candidate. These
|
|
69
|
+
* must abort/fail/retry the job; they must never be swallowed as "skip and try
|
|
70
|
+
* the next candidate" (that is the failure mode that burned scheduled slots).
|
|
71
|
+
*/
|
|
72
|
+
export type JobLevelOutage = 'database_unavailable' | 'pixiv_auth_failure' | 'delivery_unavailable' | 'network_outage';
|
|
73
|
+
/**
|
|
74
|
+
* What one failed candidate ATTEMPT means. `candidate` => the scan advances.
|
|
75
|
+
* `job` => the scan must not be allowed to degrade the run into an empty
|
|
76
|
+
* candidate list.
|
|
77
|
+
*/
|
|
78
|
+
export type CandidateFailure = {
|
|
79
|
+
scope: 'candidate';
|
|
80
|
+
skip: CandidateSkip;
|
|
81
|
+
} | {
|
|
82
|
+
scope: 'job';
|
|
83
|
+
outage: JobLevelOutage;
|
|
84
|
+
error: string;
|
|
85
|
+
};
|
|
86
|
+
/** What happened to ONE candidate work during a target's bounded scan. */
|
|
87
|
+
export type CandidateAttempt =
|
|
88
|
+
/** The candidate produced the target's business result (or a delivery intent). */
|
|
89
|
+
{
|
|
90
|
+
kind: 'selected';
|
|
91
|
+
workId: string;
|
|
92
|
+
workType: WorkType;
|
|
93
|
+
}
|
|
94
|
+
/** The candidate was unusable; the scan continues with the next one. */
|
|
95
|
+
| {
|
|
96
|
+
kind: 'skipped';
|
|
97
|
+
skip: CandidateSkip;
|
|
98
|
+
};
|
|
99
|
+
/**
|
|
100
|
+
* Bookkeeping for a target's bounded candidate scan. Attached to the terminal
|
|
101
|
+
* target outcome so the run can state exactly one of:
|
|
102
|
+
*
|
|
103
|
+
* "submitted candidate X after skipping Y candidate(s)"
|
|
104
|
+
* "no eligible candidate found after scanning N"
|
|
105
|
+
*
|
|
106
|
+
* `bound` is the configured cap (never exceeded), `attempted` the real count.
|
|
107
|
+
*/
|
|
108
|
+
export interface CandidateScanSummary {
|
|
109
|
+
/**
|
|
110
|
+
* Cap on candidates this scan was allowed to attempt: the effective window,
|
|
111
|
+
* never above the configured `candidateScanLimit` unless the target's own
|
|
112
|
+
* `limit` is larger (a multi-work target must be able to fill its limit).
|
|
113
|
+
*/
|
|
114
|
+
bound: number;
|
|
115
|
+
/** Candidates actually attempted. Always <= bound. */
|
|
116
|
+
attempted: number;
|
|
117
|
+
/**
|
|
118
|
+
* Candidates skipped before the scan ended, in the order they were skipped —
|
|
119
|
+
* strictly candidate order whenever the scan is serial, which is always the
|
|
120
|
+
* case for a single-work cell.
|
|
121
|
+
*/
|
|
122
|
+
skipped: CandidateSkip[];
|
|
123
|
+
/** Job-level outages observed while scanning (never a candidate verdict). */
|
|
124
|
+
outages: JobLevelOutage[];
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* A scan that attempted nothing. The real pipeline always reports its own scan;
|
|
128
|
+
* this is for callers/tests that have no candidate-level information, so they
|
|
129
|
+
* cannot fabricate a verdict they did not observe.
|
|
130
|
+
*/
|
|
131
|
+
export declare function emptyCandidateScan(): CandidateScanSummary;
|
|
10
132
|
export type TargetOutcome = {
|
|
11
133
|
kind: 'submitted';
|
|
12
134
|
workId: string;
|
|
13
135
|
workType: WorkType;
|
|
14
136
|
/** Durable delivery row that recorded the downstream ACK. */
|
|
15
137
|
deliveryId?: string;
|
|
138
|
+
/** Which candidates were skipped to reach this one. */
|
|
139
|
+
scan?: CandidateScanSummary;
|
|
16
140
|
}
|
|
17
141
|
/**
|
|
18
142
|
* The work was processed locally but there is no downstream delivery target
|
|
@@ -24,6 +148,7 @@ export type TargetOutcome = {
|
|
|
24
148
|
kind: 'stored';
|
|
25
149
|
workId: string;
|
|
26
150
|
workType: WorkType;
|
|
151
|
+
scan?: CandidateScanSummary;
|
|
27
152
|
}
|
|
28
153
|
/**
|
|
29
154
|
* A delivery intent + outbox item were created durably, but the downstream
|
|
@@ -35,29 +160,97 @@ export type TargetOutcome = {
|
|
|
35
160
|
workId: string;
|
|
36
161
|
workType: WorkType;
|
|
37
162
|
deliveryId: string;
|
|
163
|
+
scan?: CandidateScanSummary;
|
|
38
164
|
}
|
|
39
|
-
/**
|
|
165
|
+
/**
|
|
166
|
+
* The bounded scan found no ELIGIBLE candidate: every attempted candidate was
|
|
167
|
+
* skipped for a candidate-level reason (duplicate / deleted / denied / ...).
|
|
168
|
+
* A clean no-op, not a failure and not a retry loop.
|
|
169
|
+
*/
|
|
40
170
|
| {
|
|
41
171
|
kind: 'no_candidate';
|
|
42
172
|
reason: string;
|
|
173
|
+
scan?: CandidateScanSummary;
|
|
43
174
|
}
|
|
44
175
|
/**
|
|
45
176
|
* Downstream proved this work was already delivered by a DIFFERENT intent
|
|
46
177
|
* (historical drift). This is a terminal business duplicate, not a new
|
|
47
|
-
* submission. Produced only by the explicit reconciliation path
|
|
178
|
+
* submission. Produced only by the explicit reconciliation path
|
|
179
|
+
* (`settleDeliveryTerminal`) or by a single-work cell RESUME whose locked work
|
|
180
|
+
* turns out to be already delivered — NEVER as the verdict of a candidate scan.
|
|
48
181
|
*/
|
|
49
182
|
| {
|
|
50
183
|
kind: 'duplicate';
|
|
51
184
|
workId: string;
|
|
52
185
|
reason: string;
|
|
186
|
+
scan?: CandidateScanSummary;
|
|
53
187
|
}
|
|
54
188
|
/** The target failed. retryable=true => a later trigger/outbox may resume. */
|
|
55
189
|
| {
|
|
56
190
|
kind: 'failed';
|
|
57
191
|
retryable: boolean;
|
|
58
192
|
error: string;
|
|
193
|
+
scan?: CandidateScanSummary;
|
|
59
194
|
};
|
|
60
195
|
/** Terminal outcomes settle the cell; others leave it recoverable. */
|
|
61
196
|
export declare function isTerminalOutcome(outcome: TargetOutcome): boolean;
|
|
197
|
+
/** True when the scan ended on a candidate it may actually submit. */
|
|
198
|
+
export declare function isSelectedAttempt(attempt: CandidateAttempt | void): attempt is {
|
|
199
|
+
kind: 'selected';
|
|
200
|
+
workId: string;
|
|
201
|
+
workType: WorkType;
|
|
202
|
+
};
|
|
203
|
+
/**
|
|
204
|
+
* True when this candidate failure means "move on to the next candidate"
|
|
205
|
+
* WITHOUT retrying this one — the cause is a fact about the work (deleted,
|
|
206
|
+
* private, wrong media/format, filtered by rules, already submitted).
|
|
207
|
+
*
|
|
208
|
+
* Transient infrastructure failures return FALSE on purpose: the existing
|
|
209
|
+
* retry/backoff semantics own those, because retrying the same work is what
|
|
210
|
+
* this repo does, and churning through a whole ranking page while the network
|
|
211
|
+
* is down would be worse than failing the target once.
|
|
212
|
+
*/
|
|
213
|
+
export declare function skipCandidateWithoutRetry(skip: CandidateSkip): boolean;
|
|
214
|
+
/**
|
|
215
|
+
* The explicit, human-readable verdict of a target. Requirement: a run must
|
|
216
|
+
* report one of "submitted candidate X after skipping Y candidates" or "no
|
|
217
|
+
* eligible candidate found after scanning N" — never a bare "completed".
|
|
218
|
+
*/
|
|
62
219
|
export declare function outcomeSummary(outcome: TargetOutcome): string;
|
|
220
|
+
/**
|
|
221
|
+
* `no eligible candidate found after scanning N` (+ what was skipped).
|
|
222
|
+
*
|
|
223
|
+
* Candidates dropped before any download attempt (already submitted / already
|
|
224
|
+
* in history) are counted separately from the ones actually attempted, because
|
|
225
|
+
* "scanned 0" with three duplicates is a very different statement from
|
|
226
|
+
* "scanned 3, all unusable" — and the operator needs to be able to tell them
|
|
227
|
+
* apart.
|
|
228
|
+
*/
|
|
229
|
+
export declare function noEligibleCandidateText(scan: CandidateScanSummary): string;
|
|
230
|
+
/**
|
|
231
|
+
* True when the scan saw a transient infrastructure failure, or ANY attempted
|
|
232
|
+
* candidate failed transiently. Such a scan says "the network/API is flaky",
|
|
233
|
+
* NOT "no eligible candidate", so the caller must fail/retry the target instead
|
|
234
|
+
* of reporting a clean empty scan.
|
|
235
|
+
*/
|
|
236
|
+
export declare function hasTransientFailure(scan: CandidateScanSummary): boolean;
|
|
237
|
+
/**
|
|
238
|
+
* Fold two scans of the same logical target into one, so a multi-page or
|
|
239
|
+
* lookback scan reports the whole picture rather than only its last page.
|
|
240
|
+
* `bound` is the sum of the windows actually offered (each page is bounded
|
|
241
|
+
* independently), which keeps `attempted <= bound` true.
|
|
242
|
+
*/
|
|
243
|
+
export declare function mergeScanSummaries(first: CandidateScanSummary | null, second: CandidateScanSummary): CandidateScanSummary;
|
|
244
|
+
/**
|
|
245
|
+
* Decide what a failed candidate ATTEMPT means.
|
|
246
|
+
*
|
|
247
|
+
* This is the boundary the previous design got wrong: a duplicate was treated
|
|
248
|
+
* as a completed job. The order below is what makes the distinction real —
|
|
249
|
+
* a dead database, a dead token, a dead delivery provider or a dead network is
|
|
250
|
+
* a JOB problem and is classified as such before any candidate-level pattern
|
|
251
|
+
* can claim it.
|
|
252
|
+
*/
|
|
253
|
+
export declare function classifyCandidateFailure(error: unknown, workId: string): CandidateFailure;
|
|
254
|
+
/** Hard job-level outage for a directly-thrown error, or null. */
|
|
255
|
+
export declare function classifyJobLevelOutage(error: unknown): JobLevelOutage | null;
|
|
63
256
|
//# sourceMappingURL=TargetOutcome.d.ts.map
|
|
@@ -1,7 +1,23 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.emptyCandidateScan = emptyCandidateScan;
|
|
3
4
|
exports.isTerminalOutcome = isTerminalOutcome;
|
|
5
|
+
exports.isSelectedAttempt = isSelectedAttempt;
|
|
6
|
+
exports.skipCandidateWithoutRetry = skipCandidateWithoutRetry;
|
|
4
7
|
exports.outcomeSummary = outcomeSummary;
|
|
8
|
+
exports.noEligibleCandidateText = noEligibleCandidateText;
|
|
9
|
+
exports.hasTransientFailure = hasTransientFailure;
|
|
10
|
+
exports.mergeScanSummaries = mergeScanSummaries;
|
|
11
|
+
exports.classifyCandidateFailure = classifyCandidateFailure;
|
|
12
|
+
exports.classifyJobLevelOutage = classifyJobLevelOutage;
|
|
13
|
+
/**
|
|
14
|
+
* A scan that attempted nothing. The real pipeline always reports its own scan;
|
|
15
|
+
* this is for callers/tests that have no candidate-level information, so they
|
|
16
|
+
* cannot fabricate a verdict they did not observe.
|
|
17
|
+
*/
|
|
18
|
+
function emptyCandidateScan() {
|
|
19
|
+
return { bound: 0, attempted: 0, skipped: [], outages: [] };
|
|
20
|
+
}
|
|
5
21
|
/** Terminal outcomes settle the cell; others leave it recoverable. */
|
|
6
22
|
function isTerminalOutcome(outcome) {
|
|
7
23
|
return (outcome.kind === 'submitted' ||
|
|
@@ -10,19 +26,202 @@ function isTerminalOutcome(outcome) {
|
|
|
10
26
|
outcome.kind === 'duplicate' ||
|
|
11
27
|
(outcome.kind === 'failed' && !outcome.retryable));
|
|
12
28
|
}
|
|
29
|
+
/** True when the scan ended on a candidate it may actually submit. */
|
|
30
|
+
function isSelectedAttempt(attempt) {
|
|
31
|
+
return attempt?.kind === 'selected';
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* True when this candidate failure means "move on to the next candidate"
|
|
35
|
+
* WITHOUT retrying this one — the cause is a fact about the work (deleted,
|
|
36
|
+
* private, wrong media/format, filtered by rules, already submitted).
|
|
37
|
+
*
|
|
38
|
+
* Transient infrastructure failures return FALSE on purpose: the existing
|
|
39
|
+
* retry/backoff semantics own those, because retrying the same work is what
|
|
40
|
+
* this repo does, and churning through a whole ranking page while the network
|
|
41
|
+
* is down would be worse than failing the target once.
|
|
42
|
+
*/
|
|
43
|
+
function skipCandidateWithoutRetry(skip) {
|
|
44
|
+
return skip.retryable !== true;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* The explicit, human-readable verdict of a target. Requirement: a run must
|
|
48
|
+
* report one of "submitted candidate X after skipping Y candidates" or "no
|
|
49
|
+
* eligible candidate found after scanning N" — never a bare "completed".
|
|
50
|
+
*/
|
|
13
51
|
function outcomeSummary(outcome) {
|
|
14
52
|
switch (outcome.kind) {
|
|
15
53
|
case 'submitted':
|
|
16
54
|
case 'stored':
|
|
17
|
-
return outcome.
|
|
55
|
+
return outcome.scan
|
|
56
|
+
? `submitted candidate ${outcome.workId} (${outcome.workType}) after skipping ` +
|
|
57
|
+
`${outcome.scan.skipped.length} candidate(s)${skipDetail(outcome.scan)}`
|
|
58
|
+
: `${outcome.kind} ${outcome.workId}`;
|
|
18
59
|
case 'delivery_pending':
|
|
19
|
-
return
|
|
60
|
+
return outcome.scan
|
|
61
|
+
? `submitted candidate ${outcome.workId} (${outcome.workType}) for review after skipping ` +
|
|
62
|
+
`${outcome.scan.skipped.length} candidate(s)${skipDetail(outcome.scan)}`
|
|
63
|
+
: `delivery_pending ${outcome.workId}`;
|
|
20
64
|
case 'no_candidate':
|
|
21
|
-
return
|
|
65
|
+
return outcome.scan
|
|
66
|
+
? noEligibleCandidateText(outcome.scan)
|
|
67
|
+
: `no_candidate: ${outcome.reason}`;
|
|
22
68
|
case 'duplicate':
|
|
23
69
|
return `duplicate: ${outcome.reason}`;
|
|
24
70
|
case 'failed':
|
|
25
71
|
return `failed(${outcome.retryable ? 'retryable' : 'permanent'}): ${outcome.error}`;
|
|
26
72
|
}
|
|
27
73
|
}
|
|
74
|
+
/**
|
|
75
|
+
* `no eligible candidate found after scanning N` (+ what was skipped).
|
|
76
|
+
*
|
|
77
|
+
* Candidates dropped before any download attempt (already submitted / already
|
|
78
|
+
* in history) are counted separately from the ones actually attempted, because
|
|
79
|
+
* "scanned 0" with three duplicates is a very different statement from
|
|
80
|
+
* "scanned 3, all unusable" — and the operator needs to be able to tell them
|
|
81
|
+
* apart.
|
|
82
|
+
*/
|
|
83
|
+
function noEligibleCandidateText(scan) {
|
|
84
|
+
const scanned = scan.attempted;
|
|
85
|
+
if (scan.skipped.length === 0) {
|
|
86
|
+
return `no eligible candidate found after scanning ${scanned}`;
|
|
87
|
+
}
|
|
88
|
+
const codes = [...new Set(scan.skipped.map((s) => s.code))].join(', ');
|
|
89
|
+
const prefiltered = Math.max(0, scan.skipped.length - scanned);
|
|
90
|
+
const detail = prefiltered > 0
|
|
91
|
+
? ` (bound ${scan.bound}); all ${scan.skipped.length} candidate(s) unusable ` +
|
|
92
|
+
`[${prefiltered} filtered before download, ${scanned} attempted]: ${codes}`
|
|
93
|
+
: ` (bound ${scan.bound}); all ${scan.skipped.length} attempted candidate(s) skipped: ${codes}`;
|
|
94
|
+
return `no eligible candidate found after scanning ${scanned}${detail}${skipDetail(scan)}`;
|
|
95
|
+
}
|
|
96
|
+
function skipDetail(scan) {
|
|
97
|
+
if (scan.skipped.length === 0)
|
|
98
|
+
return '';
|
|
99
|
+
const sample = scan.skipped
|
|
100
|
+
.slice(0, 3)
|
|
101
|
+
.map((s) => `${s.code}(${s.workId})`)
|
|
102
|
+
.join(', ');
|
|
103
|
+
return ` [${sample}${scan.skipped.length > 3 ? `, +${scan.skipped.length - 3} more` : ''}]`;
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* True when the scan saw a transient infrastructure failure, or ANY attempted
|
|
107
|
+
* candidate failed transiently. Such a scan says "the network/API is flaky",
|
|
108
|
+
* NOT "no eligible candidate", so the caller must fail/retry the target instead
|
|
109
|
+
* of reporting a clean empty scan.
|
|
110
|
+
*/
|
|
111
|
+
function hasTransientFailure(scan) {
|
|
112
|
+
if (scan.outages.length > 0)
|
|
113
|
+
return true;
|
|
114
|
+
return scan.skipped.some((skip) => skip.retryable === true);
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Fold two scans of the same logical target into one, so a multi-page or
|
|
118
|
+
* lookback scan reports the whole picture rather than only its last page.
|
|
119
|
+
* `bound` is the sum of the windows actually offered (each page is bounded
|
|
120
|
+
* independently), which keeps `attempted <= bound` true.
|
|
121
|
+
*/
|
|
122
|
+
function mergeScanSummaries(first, second) {
|
|
123
|
+
if (!first)
|
|
124
|
+
return second;
|
|
125
|
+
return {
|
|
126
|
+
bound: first.bound + second.bound,
|
|
127
|
+
attempted: first.attempted + second.attempted,
|
|
128
|
+
skipped: [...first.skipped, ...second.skipped],
|
|
129
|
+
outages: [...new Set([...first.outages, ...second.outages])],
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
const SQLITE_OUTAGE = /sqlite|database (?:is )?locked|unable to open database|no such table|disk i\/o error/i;
|
|
133
|
+
/** A delivery-provider OUTAGE — not a per-message rejection such as a too-long caption. */
|
|
134
|
+
const DELIVERY_OUTAGE = /(?:telegram|delivery (?:target|provider))[^.]{0,80}\b(?:unavailable|unreachable|down|not configured|timed? ?out|econn\w*|etimedout|enotfound|401|403|50[234])\b|api\.telegram\.org[^.]*\b(?:econn\w*|etimedout|enotfound|50[234])\b/i;
|
|
135
|
+
const AUTH_OUTAGE = /\b(401|unauthorized|invalid_grant|invalid refresh token|authentication failed)\b/i;
|
|
136
|
+
const NETWORK_OUTAGE = /econnrefused|econnreset|enotfound|etimedout|ehostunreach|enetunreach|socket hang up|network is unreachable|getaddrinfo/i;
|
|
137
|
+
const RATE_LIMIT = /\b429\b|rate limit/i;
|
|
138
|
+
function messageOf(error) {
|
|
139
|
+
if (error instanceof Error) {
|
|
140
|
+
return `${error.name}: ${error.message}`;
|
|
141
|
+
}
|
|
142
|
+
return String(error);
|
|
143
|
+
}
|
|
144
|
+
function statusOf(error) {
|
|
145
|
+
const status = error?.status ??
|
|
146
|
+
error?.statusCode;
|
|
147
|
+
return typeof status === 'number' ? status : undefined;
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* Decide what a failed candidate ATTEMPT means.
|
|
151
|
+
*
|
|
152
|
+
* This is the boundary the previous design got wrong: a duplicate was treated
|
|
153
|
+
* as a completed job. The order below is what makes the distinction real —
|
|
154
|
+
* a dead database, a dead token, a dead delivery provider or a dead network is
|
|
155
|
+
* a JOB problem and is classified as such before any candidate-level pattern
|
|
156
|
+
* can claim it.
|
|
157
|
+
*/
|
|
158
|
+
function classifyCandidateFailure(error, workId) {
|
|
159
|
+
const message = messageOf(error);
|
|
160
|
+
const status = statusOf(error);
|
|
161
|
+
const name = error instanceof Error ? error.name : '';
|
|
162
|
+
const code = error?.code;
|
|
163
|
+
const codeText = typeof code === 'string' ? code : '';
|
|
164
|
+
// --- Job-level first: these must never shrink to "empty candidate list" ---
|
|
165
|
+
if (name === 'DatabaseError' || codeText.startsWith('SQLITE_') || SQLITE_OUTAGE.test(message)) {
|
|
166
|
+
return { scope: 'job', outage: 'database_unavailable', error: message };
|
|
167
|
+
}
|
|
168
|
+
if (name === 'AuthenticationError' || status === 401 || AUTH_OUTAGE.test(message)) {
|
|
169
|
+
return { scope: 'job', outage: 'pixiv_auth_failure', error: message };
|
|
170
|
+
}
|
|
171
|
+
// The delivery provider being unreachable says nothing about the candidate:
|
|
172
|
+
// every candidate would "fail" identically, so this must never be counted as
|
|
173
|
+
// an empty candidate list.
|
|
174
|
+
if (DELIVERY_OUTAGE.test(message) && !/already delivered|already published|duplicate/i.test(message)) {
|
|
175
|
+
return { scope: 'job', outage: 'delivery_unavailable', error: message };
|
|
176
|
+
}
|
|
177
|
+
if (status === 502 || status === 503 || status === 504) {
|
|
178
|
+
// The remote provider itself is down, not this work.
|
|
179
|
+
return { scope: 'job', outage: 'network_outage', error: message };
|
|
180
|
+
}
|
|
181
|
+
if (NETWORK_OUTAGE.test(message)) {
|
|
182
|
+
return {
|
|
183
|
+
scope: 'candidate',
|
|
184
|
+
skip: { code: 'unavailable', workId, reason: message, retryable: true },
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
// --- Candidate-level: this ONE work is unusable, try the next one ---------
|
|
188
|
+
if (/already delivered|already processed|already published|idempotent_replay/i.test(message)) {
|
|
189
|
+
return {
|
|
190
|
+
scope: 'candidate',
|
|
191
|
+
skip: { code: 'duplicate', workId, reason: message },
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
if (name === 'PixivNotFoundError' || status === 404 || /\b404\b|not found|deleted/i.test(message)) {
|
|
195
|
+
return { scope: 'candidate', skip: { code: 'deleted', workId, reason: message } };
|
|
196
|
+
}
|
|
197
|
+
if (name === 'PixivRateLimitError' || RATE_LIMIT.test(message)) {
|
|
198
|
+
return {
|
|
199
|
+
scope: 'candidate',
|
|
200
|
+
skip: { code: 'unavailable', workId, reason: message, retryable: true },
|
|
201
|
+
};
|
|
202
|
+
}
|
|
203
|
+
if (status === 403 || /forbidden|private|access denied|permission/i.test(message)) {
|
|
204
|
+
return { scope: 'candidate', skip: { code: 'access_denied', workId, reason: message } };
|
|
205
|
+
}
|
|
206
|
+
if (/ugoira|unsupported (?:media|type|format)|cannot (?:process|handle)/i.test(message)) {
|
|
207
|
+
return { scope: 'candidate', skip: { code: 'unsupported_media', workId, reason: message } };
|
|
208
|
+
}
|
|
209
|
+
if (/language filter|filtered out|excluded (?:by|from)/i.test(message)) {
|
|
210
|
+
return { scope: 'candidate', skip: { code: 'filtered', workId, reason: message } };
|
|
211
|
+
}
|
|
212
|
+
if (/invalid (?:metadata|id|illustId|novelId)|missing (?:metadata|page|image)|no files produced/i.test(message)) {
|
|
213
|
+
return { scope: 'candidate', skip: { code: 'invalid_metadata', workId, reason: message } };
|
|
214
|
+
}
|
|
215
|
+
// Unclassifiable: treat as a transient candidate problem (retryable) so an
|
|
216
|
+
// all-unknown scan fails the job rather than silently reporting an empty scan.
|
|
217
|
+
return {
|
|
218
|
+
scope: 'candidate',
|
|
219
|
+
skip: { code: 'unavailable', workId, reason: message, retryable: true },
|
|
220
|
+
};
|
|
221
|
+
}
|
|
222
|
+
/** Hard job-level outage for a directly-thrown error, or null. */
|
|
223
|
+
function classifyJobLevelOutage(error) {
|
|
224
|
+
const failure = classifyCandidateFailure(error, '');
|
|
225
|
+
return failure.scope === 'job' ? failure.outage : null;
|
|
226
|
+
}
|
|
28
227
|
//# sourceMappingURL=TargetOutcome.js.map
|