pixivflow 2.19.4 → 2.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/SchedulerCommand.js +36 -0
- package/dist/commands/SchedulerRunOnceCommand.d.ts +2 -3
- package/dist/commands/SchedulerRunOnceCommand.js +2 -3
- package/dist/commands/scheduler-runtime.js +17 -1
- package/dist/config/defaults.d.ts +1 -0
- package/dist/config/defaults.js +4 -0
- package/dist/config/environment.js +19 -0
- package/dist/config/types.d.ts +28 -0
- package/dist/config/validation.js +25 -0
- package/dist/delivery/DeliveryService.d.ts +30 -1
- package/dist/delivery/DeliveryService.js +13 -2
- package/dist/delivery/HttpMultipartDelivery.js +29 -8
- package/dist/delivery/OutboxWorker.d.ts +2 -0
- package/dist/delivery/OutboxWorker.js +3 -0
- package/dist/delivery/types.d.ts +18 -0
- package/dist/download/DownloadManager.js +12 -1
- package/dist/download/handlers/IllustrationTargetHandler.d.ts +42 -2
- package/dist/download/handlers/IllustrationTargetHandler.js +240 -50
- package/dist/download/handlers/NovelTargetHandler.d.ts +47 -0
- package/dist/download/handlers/NovelTargetHandler.js +213 -27
- package/dist/download/handlers/deliveryContext.d.ts +16 -0
- package/dist/download/handlers/deliveryContext.js +32 -0
- package/dist/download/pipeline/DownloadPipeline.d.ts +22 -3
- package/dist/download/pipeline/DownloadPipeline.js +109 -92
- package/dist/download/plan/DownloadPlanner.d.ts +47 -1
- package/dist/download/plan/DownloadPlanner.js +105 -15
- package/dist/notification/NotificationPolicy.d.ts +12 -0
- package/dist/notification/NotificationPolicy.js +86 -0
- package/dist/package.json +1 -1
- package/dist/scheduler/MultiScheduleManager.js +2 -0
- package/dist/scheduler/ScheduleTriggerServer.d.ts +58 -1
- package/dist/scheduler/ScheduleTriggerServer.js +207 -40
- package/dist/scheduler/SlotCoordinator.d.ts +80 -2
- package/dist/scheduler/SlotCoordinator.js +145 -3
- package/dist/scheduler/TargetOutcome.d.ts +195 -2
- package/dist/scheduler/TargetOutcome.js +202 -3
- package/dist/storage/DatabaseMigration.js +11 -0
- package/dist/storage/repositories/DeliveryRepository.d.ts +12 -0
- package/dist/storage/repositories/DeliveryRepository.js +18 -1
- package/dist/storage/repositories/SlotRepository.d.ts +8 -0
- package/dist/storage/repositories/SlotRepository.js +6 -2
- package/dist/version.js +1 -1
- package/dist/webui/package.json +1 -1
- package/package.json +1 -1
|
@@ -157,6 +157,42 @@ class SchedulerCommand extends Command_1.BaseCommand {
|
|
|
157
157
|
};
|
|
158
158
|
},
|
|
159
159
|
drainOutbox: () => runtime.drainOutbox(),
|
|
160
|
+
refetch: async (targetId, requestId, correlationId) => {
|
|
161
|
+
const cfg = resolveConfig();
|
|
162
|
+
const plans = (cfg.schedules ?? []).filter((plan) => plan.enabled !== false && (0, schedules_1.selectScheduleTargets)(cfg.targets, plan).some((target) => target.id === targetId));
|
|
163
|
+
if (plans.length === 0)
|
|
164
|
+
throw new Error('unknown target');
|
|
165
|
+
if (plans.length !== 1)
|
|
166
|
+
throw new Error('ambiguous target');
|
|
167
|
+
const plan = plans[0];
|
|
168
|
+
const target = (0, schedules_1.selectScheduleTargets)(cfg.targets, plan).find((item) => item.id === targetId);
|
|
169
|
+
const now = new Date();
|
|
170
|
+
const date = new Intl.DateTimeFormat('en-CA', {
|
|
171
|
+
timeZone: plan.timezone ?? 'UTC', year: 'numeric', month: '2-digit', day: '2-digit',
|
|
172
|
+
}).format(now);
|
|
173
|
+
const slot = {
|
|
174
|
+
slotId: `${plan.id}@manual-${requestId.toLowerCase()}`,
|
|
175
|
+
scheduleId: plan.id,
|
|
176
|
+
occurrenceAt: now.getTime(),
|
|
177
|
+
occurrenceDate: date,
|
|
178
|
+
occurrenceLabel: 'manual',
|
|
179
|
+
timezone: plan.timezone ?? 'UTC',
|
|
180
|
+
triggerSource: 'manual',
|
|
181
|
+
slotName: '审核群重抓',
|
|
182
|
+
slotDate: date,
|
|
183
|
+
manualRequestId: requestId,
|
|
184
|
+
correlationId: correlationId || undefined,
|
|
185
|
+
};
|
|
186
|
+
const existing = runtime.database.slots.getSlot(slot.slotId);
|
|
187
|
+
if (existing && (existing.scheduleId !== plan.id || existing.targetIds.length !== 1 || existing.targetIds[0] !== targetId)) {
|
|
188
|
+
throw new Error('ambiguous target');
|
|
189
|
+
}
|
|
190
|
+
const prepared = coordinator.prepare(slot, plan, [target]);
|
|
191
|
+
if (prepared.alreadyCompleted)
|
|
192
|
+
return { slotId: slot.slotId, disposition: 'already_completed' };
|
|
193
|
+
const started = manager.triggerSchedule(plan.id, { slot, onlyTarget: targetId, triggerSource: 'manual' });
|
|
194
|
+
return { slotId: slot.slotId, disposition: started ? 'accepted' : 'queued' };
|
|
195
|
+
},
|
|
160
196
|
status: (scheduleId) => {
|
|
161
197
|
const cfg = resolveConfig();
|
|
162
198
|
const plan = findPlan(cfg, scheduleId);
|
|
@@ -2,9 +2,8 @@
|
|
|
2
2
|
* Scheduler run-once command
|
|
3
3
|
*
|
|
4
4
|
* Runs every enabled schedule's download plan exactly once, then exits.
|
|
5
|
-
* This is the
|
|
6
|
-
* a
|
|
7
|
-
* so refetch needs a bounded one-shot invocation instead.
|
|
5
|
+
* This is the local CLI path. Remote review-group refetch uses the authenticated
|
|
6
|
+
* trigger server and a durable manual Slot so a sleeping worker can recover it.
|
|
8
7
|
*/
|
|
9
8
|
import { BaseCommand } from './Command';
|
|
10
9
|
import { CommandCategory } from './metadata';
|
|
@@ -3,9 +3,8 @@
|
|
|
3
3
|
* Scheduler run-once command
|
|
4
4
|
*
|
|
5
5
|
* Runs every enabled schedule's download plan exactly once, then exits.
|
|
6
|
-
* This is the
|
|
7
|
-
* a
|
|
8
|
-
* so refetch needs a bounded one-shot invocation instead.
|
|
6
|
+
* This is the local CLI path. Remote review-group refetch uses the authenticated
|
|
7
|
+
* trigger server and a durable manual Slot so a sleeping worker can recover it.
|
|
9
8
|
*/
|
|
10
9
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
11
10
|
exports.SchedulerRunOnceCommand = void 0;
|
|
@@ -438,10 +438,26 @@ async function createSchedulerRuntime(configPathArg) {
|
|
|
438
438
|
if (!target.id)
|
|
439
439
|
return;
|
|
440
440
|
options.onTargetOutcome?.(target.id, outcome);
|
|
441
|
-
if (!scheduleSlot)
|
|
441
|
+
if (!scheduleSlot) {
|
|
442
|
+
// No durable slot (run-once CLI): nothing to converge or report.
|
|
442
443
|
return;
|
|
444
|
+
}
|
|
443
445
|
coordinator.applyOutcome(scheduleSlot.slotId, target.id, outcome);
|
|
444
446
|
notificationPolicy.noteOutcome(scheduleSlot.slotId, scheduleSlot, schedule, target, outcome);
|
|
447
|
+
// A remote manual replacement ("重抓") must report its terminal verdict
|
|
448
|
+
// back to the reviewer. Only terminal outcomes are reported: a candidate
|
|
449
|
+
// the scan skipped is not a verdict, and a durable delivery intent
|
|
450
|
+
// ('delivery_pending' / later 'submitted') is reported through the
|
|
451
|
+
// replacement submission itself (the caller correlates on requestId).
|
|
452
|
+
const manualRequestId = scheduleSlot.manualRequestId;
|
|
453
|
+
if (manualRequestId) {
|
|
454
|
+
const terminal = outcome.kind === 'no_candidate' ||
|
|
455
|
+
outcome.kind === 'duplicate' ||
|
|
456
|
+
(outcome.kind === 'failed' && !outcome.retryable);
|
|
457
|
+
if (terminal) {
|
|
458
|
+
notificationPolicy.noteRefetchOutcome(scheduleSlot, schedule, target, manualRequestId, outcome);
|
|
459
|
+
}
|
|
460
|
+
}
|
|
445
461
|
});
|
|
446
462
|
if (scheduleSlot) {
|
|
447
463
|
downloadManager.slotContext = {
|
package/dist/config/defaults.js
CHANGED
|
@@ -41,6 +41,10 @@ exports.DEFAULT_CONFIG = {
|
|
|
41
41
|
maxRetries: 3,
|
|
42
42
|
retryDelay: 2000,
|
|
43
43
|
timeout: 60000,
|
|
44
|
+
// Candidates one execution may ATTEMPT while looking for an eligible work.
|
|
45
|
+
// Bounded so a ranking page full of already-delivered works cannot loop,
|
|
46
|
+
// but large enough that a single duplicate no longer burns the slot.
|
|
47
|
+
candidateScanLimit: 5,
|
|
44
48
|
},
|
|
45
49
|
initialDelay: 0,
|
|
46
50
|
};
|
|
@@ -101,6 +101,25 @@ function applyEnvironmentOverrides(config) {
|
|
|
101
101
|
overridden.scheduler.enabled = process.env.PIXIV_SCHEDULER_ENABLED.toLowerCase() === 'true';
|
|
102
102
|
}
|
|
103
103
|
}
|
|
104
|
+
// Override the bounded candidate-scan window. A non-numeric or non-positive
|
|
105
|
+
// value is ignored (the config-file value stays authoritative) rather than
|
|
106
|
+
// silently becoming NaN, which would disable the bound entirely.
|
|
107
|
+
if (process.env.PIXIV_CANDIDATE_SCAN_LIMIT !== undefined) {
|
|
108
|
+
const parsed = Number.parseInt(process.env.PIXIV_CANDIDATE_SCAN_LIMIT, 10);
|
|
109
|
+
if (Number.isFinite(parsed) && parsed > 0) {
|
|
110
|
+
if (!overridden.download) {
|
|
111
|
+
overridden.download = { candidateScanLimit: parsed };
|
|
112
|
+
}
|
|
113
|
+
else {
|
|
114
|
+
overridden.download.candidateScanLimit = parsed;
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
else {
|
|
118
|
+
logger_1.logger.warn('Ignoring PIXIV_CANDIDATE_SCAN_LIMIT: expected a positive integer', {
|
|
119
|
+
value: process.env.PIXIV_CANDIDATE_SCAN_LIMIT,
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
}
|
|
104
123
|
// Override proxy from environment variables
|
|
105
124
|
// Priority: all_proxy > https_proxy > http_proxy
|
|
106
125
|
const proxyUrl = process.env.all_proxy || process.env.ALL_PROXY ||
|
package/dist/config/types.d.ts
CHANGED
|
@@ -81,6 +81,20 @@ export interface TargetConfig {
|
|
|
81
81
|
* Maximum works to download per execution for this tag.
|
|
82
82
|
*/
|
|
83
83
|
limit?: number;
|
|
84
|
+
/**
|
|
85
|
+
* Maximum CANDIDATE works one execution may attempt (not produce) while
|
|
86
|
+
* looking for an eligible work. A scheduled "one post per slot" target has
|
|
87
|
+
* `limit: 1`, so without this the run would hold a single candidate: if that
|
|
88
|
+
* one candidate is already delivered / deleted / private, the slot ends with
|
|
89
|
+
* nothing submitted. The scan walks up to N candidates in ranking order,
|
|
90
|
+
* skipping unusable ones, and is strictly bounded so a page full of
|
|
91
|
+
* duplicates cannot loop.
|
|
92
|
+
*
|
|
93
|
+
* Overrides `download.candidateScanLimit` / `PIXIV_CANDIDATE_SCAN_LIMIT`.
|
|
94
|
+
* Clamped to 1..100 and never below `limit`, so a multi-work target can still
|
|
95
|
+
* fill its own limit.
|
|
96
|
+
*/
|
|
97
|
+
candidateScanLimit?: number;
|
|
84
98
|
/**
|
|
85
99
|
* Search target parameter for Pixiv API.
|
|
86
100
|
*/
|
|
@@ -503,6 +517,13 @@ export interface HttpMultipartDeliveryConfig {
|
|
|
503
517
|
url: string;
|
|
504
518
|
/** Optional JSON endpoint used for no-match operational notifications. */
|
|
505
519
|
notificationUrl?: string;
|
|
520
|
+
/**
|
|
521
|
+
* Optional JSON endpoint for remote manual replacement ("重抓") terminal
|
|
522
|
+
* verdicts (no_alternative / failed). Must be http(s); if unset, manual
|
|
523
|
+
* outcome reports are skipped (the work itself is still durable). Reuses
|
|
524
|
+
* `headers` for auth, so no extra credential is needed.
|
|
525
|
+
*/
|
|
526
|
+
refetchOutcomeUrl?: string;
|
|
506
527
|
method?: 'POST' | 'PUT';
|
|
507
528
|
/** 支持 ${ENV_NAME} 环境变量插值 */
|
|
508
529
|
headers?: Record<string, string>;
|
|
@@ -655,6 +676,13 @@ export interface StandaloneConfig {
|
|
|
655
676
|
* Default: 60000
|
|
656
677
|
*/
|
|
657
678
|
timeout?: number;
|
|
679
|
+
/**
|
|
680
|
+
* How many CANDIDATE works one execution may attempt while looking for an
|
|
681
|
+
* eligible one. Per-target `candidateScanLimit` overrides this.
|
|
682
|
+
*
|
|
683
|
+
* Default: 5 (clamped to 1..100)
|
|
684
|
+
*/
|
|
685
|
+
candidateScanLimit?: number;
|
|
658
686
|
};
|
|
659
687
|
}
|
|
660
688
|
//# sourceMappingURL=types.d.ts.map
|
|
@@ -302,6 +302,16 @@ function validateConfig(config, location, databasePath) {
|
|
|
302
302
|
errors.push(`${prefix}.notificationUrl: Must be a valid HTTP or HTTPS URL`);
|
|
303
303
|
}
|
|
304
304
|
}
|
|
305
|
+
if (delivery.refetchOutcomeUrl && !/\$\{[A-Za-z_][A-Za-z0-9_]*\}/.test(delivery.refetchOutcomeUrl)) {
|
|
306
|
+
try {
|
|
307
|
+
const url = new URL(delivery.refetchOutcomeUrl);
|
|
308
|
+
if (!['http:', 'https:'].includes(url.protocol))
|
|
309
|
+
throw new Error('unsupported protocol');
|
|
310
|
+
}
|
|
311
|
+
catch {
|
|
312
|
+
errors.push(`${prefix}.refetchOutcomeUrl: Must be a valid HTTP or HTTPS URL`);
|
|
313
|
+
}
|
|
314
|
+
}
|
|
305
315
|
if (delivery.readinessUrl && !/\$\{[A-Za-z_][A-Za-z0-9_]*\}/.test(delivery.readinessUrl)) {
|
|
306
316
|
try {
|
|
307
317
|
const url = new URL(delivery.readinessUrl);
|
|
@@ -469,6 +479,21 @@ function validateConfig(config, location, databasePath) {
|
|
|
469
479
|
if (config.download.maxRetries !== undefined && (config.download.maxRetries < 0 || config.download.maxRetries > 10)) {
|
|
470
480
|
warnings.push('download.maxRetries: Should be between 0 and 10');
|
|
471
481
|
}
|
|
482
|
+
if (config.download.candidateScanLimit !== undefined &&
|
|
483
|
+
(!Number.isInteger(config.download.candidateScanLimit) ||
|
|
484
|
+
config.download.candidateScanLimit < 1 ||
|
|
485
|
+
config.download.candidateScanLimit > 100)) {
|
|
486
|
+
warnings.push('download.candidateScanLimit: Should be an integer between 1 and 100');
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
// The bounded candidate scan is what stops a page full of duplicates from
|
|
490
|
+
// burning a whole scheduled slot, so a non-integer value is reported here.
|
|
491
|
+
for (const target of config.targets ?? []) {
|
|
492
|
+
if (target.candidateScanLimit === undefined)
|
|
493
|
+
continue;
|
|
494
|
+
if (!Number.isInteger(target.candidateScanLimit) || target.candidateScanLimit < 1 || target.candidateScanLimit > 100) {
|
|
495
|
+
warnings.push(`targets.${target.id ?? target.tag ?? '?'}.candidateScanLimit: Should be an integer between 1 and 100`);
|
|
496
|
+
}
|
|
472
497
|
}
|
|
473
498
|
// Validate log level
|
|
474
499
|
if (config.logLevel && !['debug', 'info', 'warn', 'error'].includes(config.logLevel)) {
|
|
@@ -19,6 +19,26 @@ export interface EnqueueResult {
|
|
|
19
19
|
/** True when this call created a brand new delivery row. */
|
|
20
20
|
created: boolean;
|
|
21
21
|
}
|
|
22
|
+
/**
|
|
23
|
+
* Machine-readable terminal verdict of a remote manual replacement ("重抓"),
|
|
24
|
+
* reported back to the requester (TelePost) through the durable outbox.
|
|
25
|
+
* Never contains credentials; ids are the opaque request UUID and Pixiv work ids.
|
|
26
|
+
*/
|
|
27
|
+
export interface RefetchOutcomePayload {
|
|
28
|
+
requestId: string;
|
|
29
|
+
/** 'no_alternative' | 'failed' (replacement success rides the submission). */
|
|
30
|
+
disposition: 'no_alternative' | 'failed';
|
|
31
|
+
reason?: string;
|
|
32
|
+
workId?: string;
|
|
33
|
+
/** Bounded candidate-scan bookkeeping for diagnostics (spec-compatible). */
|
|
34
|
+
scanned?: number;
|
|
35
|
+
skipped?: {
|
|
36
|
+
total: number;
|
|
37
|
+
duplicate: number;
|
|
38
|
+
invalid: number;
|
|
39
|
+
unavailable: number;
|
|
40
|
+
};
|
|
41
|
+
}
|
|
22
42
|
export declare class DeliveryService {
|
|
23
43
|
private readonly database;
|
|
24
44
|
constructor(database: Database);
|
|
@@ -35,6 +55,15 @@ export declare class DeliveryService {
|
|
|
35
55
|
isAlreadyDelivered(deliveryTarget: string, workType: string, pixivId: string): boolean;
|
|
36
56
|
/** Batch form for pre-lock candidate filtering. */
|
|
37
57
|
deliveredIds(deliveryTarget: string, workType: string, pixivIds: string[]): Set<string>;
|
|
58
|
+
/**
|
|
59
|
+
* Bulk CANDIDATE SELECTION dedupe: works already delivered to this target, or
|
|
60
|
+
* whose review submission is still PENDING an answer. A pending work is
|
|
61
|
+
* already submitted for review, so it must not be selected (and submitted)
|
|
62
|
+
* again; the scan moves on to the next candidate instead. Within-slot RESUME
|
|
63
|
+
* keeps using `isAlreadyDelivered` — a cell continuing its OWN pending work is
|
|
64
|
+
* resuming it, not duplicating it.
|
|
65
|
+
*/
|
|
66
|
+
submittedIds(deliveryTarget: string, workType: string, pixivIds: string[]): Set<string>;
|
|
38
67
|
/**
|
|
39
68
|
* Atomically create the delivery intent + outbox row and advance the cell.
|
|
40
69
|
* Missing local artifact files are treated as a recoverable error (the
|
|
@@ -67,7 +96,7 @@ export declare class DeliveryService {
|
|
|
67
96
|
created: boolean;
|
|
68
97
|
};
|
|
69
98
|
/** Enqueue a durable notification (retried independently; never affects content). */
|
|
70
|
-
enqueueNotification(deliveryTarget: string, text: string, idempotencyKey: string): void;
|
|
99
|
+
enqueueNotification(deliveryTarget: string, text: string, idempotencyKey: string, refetchOutcome?: RefetchOutcomePayload): void;
|
|
71
100
|
private guardCell;
|
|
72
101
|
private contextFrom;
|
|
73
102
|
}
|
|
@@ -33,6 +33,17 @@ class DeliveryService {
|
|
|
33
33
|
deliveredIds(deliveryTarget, workType, pixivIds) {
|
|
34
34
|
return this.database.deliveries.deliveredIds(deliveryTarget, workType, pixivIds);
|
|
35
35
|
}
|
|
36
|
+
/**
|
|
37
|
+
* Bulk CANDIDATE SELECTION dedupe: works already delivered to this target, or
|
|
38
|
+
* whose review submission is still PENDING an answer. A pending work is
|
|
39
|
+
* already submitted for review, so it must not be selected (and submitted)
|
|
40
|
+
* again; the scan moves on to the next candidate instead. Within-slot RESUME
|
|
41
|
+
* keeps using `isAlreadyDelivered` — a cell continuing its OWN pending work is
|
|
42
|
+
* resuming it, not duplicating it.
|
|
43
|
+
*/
|
|
44
|
+
submittedIds(deliveryTarget, workType, pixivIds) {
|
|
45
|
+
return this.database.deliveries.submittedIds(deliveryTarget, workType, pixivIds);
|
|
46
|
+
}
|
|
36
47
|
/**
|
|
37
48
|
* Atomically create the delivery intent + outbox row and advance the cell.
|
|
38
49
|
* Missing local artifact files are treated as a recoverable error (the
|
|
@@ -164,12 +175,12 @@ class DeliveryService {
|
|
|
164
175
|
return { deliveryId: row.id, created };
|
|
165
176
|
}
|
|
166
177
|
/** Enqueue a durable notification (retried independently; never affects content). */
|
|
167
|
-
enqueueNotification(deliveryTarget, text, idempotencyKey) {
|
|
178
|
+
enqueueNotification(deliveryTarget, text, idempotencyKey, refetchOutcome) {
|
|
168
179
|
this.database.outbox.enqueue({
|
|
169
180
|
kind: 'notification',
|
|
170
181
|
deliveryTarget,
|
|
171
182
|
idempotencyKey,
|
|
172
|
-
payload: { text },
|
|
183
|
+
payload: refetchOutcome ? { text, refetchOutcome } : { text },
|
|
173
184
|
});
|
|
174
185
|
}
|
|
175
186
|
guardCell(slotId, targetId, next) {
|
|
@@ -133,33 +133,54 @@ class HttpMultipartDelivery {
|
|
|
133
133
|
}
|
|
134
134
|
/** Single notification attempt; the outbox owns retries. */
|
|
135
135
|
async notifyOnce(request) {
|
|
136
|
-
const
|
|
137
|
-
|
|
138
|
-
|
|
136
|
+
const outcome = request.refetchOutcome;
|
|
137
|
+
const url = (outcome ? this.config.refetchOutcomeUrl?.trim() : this.config.notificationUrl?.trim());
|
|
138
|
+
if (!url) {
|
|
139
|
+
throw new Error(outcome
|
|
140
|
+
? 'HTTP delivery refetchOutcomeUrl is not configured'
|
|
141
|
+
: 'HTTP delivery notificationUrl is not configured');
|
|
142
|
+
}
|
|
139
143
|
const headers = {
|
|
140
144
|
...this.resolveHeaders(this.config.headers ?? {}),
|
|
141
145
|
'Content-Type': 'application/json',
|
|
142
146
|
};
|
|
147
|
+
// Refetch verdicts are machine-readable JSON (the requester's review state
|
|
148
|
+
// machine consumes disposition, not prose). Plain notifications remain
|
|
149
|
+
// {text, idempotency_key}.
|
|
150
|
+
const body = outcome
|
|
151
|
+
? {
|
|
152
|
+
request_id: outcome.requestId,
|
|
153
|
+
disposition: outcome.disposition,
|
|
154
|
+
reason: outcome.reason,
|
|
155
|
+
work_id: outcome.workId,
|
|
156
|
+
scanned: outcome.scanned,
|
|
157
|
+
skipped: outcome.skipped,
|
|
158
|
+
}
|
|
159
|
+
: { text: request.text, idempotency_key: request.idempotencyKey };
|
|
143
160
|
const options = {
|
|
144
161
|
method: 'POST',
|
|
145
162
|
headers,
|
|
146
|
-
body: JSON.stringify(
|
|
163
|
+
body: JSON.stringify(body),
|
|
147
164
|
};
|
|
148
165
|
if (this.dispatcher)
|
|
149
166
|
options.dispatcher = this.dispatcher;
|
|
150
167
|
const response = await fetch(this.interpolateEnvironment(url), options);
|
|
151
168
|
const text = await response.text();
|
|
152
|
-
let
|
|
169
|
+
let parsed = text;
|
|
153
170
|
if (text) {
|
|
154
171
|
try {
|
|
155
|
-
|
|
172
|
+
parsed = JSON.parse(text);
|
|
156
173
|
}
|
|
157
174
|
catch { /* plain ok */ }
|
|
158
175
|
}
|
|
159
176
|
if (!response.ok)
|
|
160
177
|
throw new Error(`notification endpoint returned HTTP ${response.status}`);
|
|
161
|
-
logger_1.logger.info('HTTP delivery notification sent', {
|
|
162
|
-
|
|
178
|
+
logger_1.logger.info('HTTP delivery notification sent', {
|
|
179
|
+
url: (0, redact_1.redactUrl)(url),
|
|
180
|
+
status: response.status,
|
|
181
|
+
hasRefetchOutcome: Boolean(outcome),
|
|
182
|
+
});
|
|
183
|
+
return { status: response.status, body: parsed };
|
|
163
184
|
}
|
|
164
185
|
async attempt(request) {
|
|
165
186
|
const fields = this.resolveFields({ ...(this.config.fields ?? {}), ...(request.fields ?? {}) }, request);
|
|
@@ -29,6 +29,8 @@ export interface DeliveryPayload {
|
|
|
29
29
|
/** Notification side-effect payload. */
|
|
30
30
|
export interface NotificationPayload {
|
|
31
31
|
text: string;
|
|
32
|
+
/** Optional structured remote-manual-replacement verdict (refetch outcome). */
|
|
33
|
+
refetchOutcome?: unknown;
|
|
32
34
|
}
|
|
33
35
|
/** Exponential backoff with jitter, capped. */
|
|
34
36
|
export declare function backoffDelayMs(attempt: number, base: number, max: number): number;
|
|
@@ -211,6 +211,9 @@ class OutboxWorker {
|
|
|
211
211
|
await this.dispatcher.notify(row.deliveryTarget, {
|
|
212
212
|
text: payload.text,
|
|
213
213
|
idempotencyKey: row.idempotencyKey ?? row.id,
|
|
214
|
+
refetchOutcome: payload.refetchOutcome !== undefined
|
|
215
|
+
? payload.refetchOutcome
|
|
216
|
+
: undefined,
|
|
214
217
|
});
|
|
215
218
|
}
|
|
216
219
|
else {
|
package/dist/delivery/types.d.ts
CHANGED
|
@@ -89,6 +89,24 @@ export interface DeliveryResult {
|
|
|
89
89
|
export interface DeliveryNotificationRequest {
|
|
90
90
|
text: string;
|
|
91
91
|
idempotencyKey: string;
|
|
92
|
+
/**
|
|
93
|
+
* Optional structured remote-manual-replacement verdict. When present the
|
|
94
|
+
* delivery sends a JSON body to the target's `refetchOutcomeUrl` instead of
|
|
95
|
+
* a plain text notification to `notificationUrl` (auth reuses `headers`).
|
|
96
|
+
*/
|
|
97
|
+
refetchOutcome?: {
|
|
98
|
+
requestId: string;
|
|
99
|
+
disposition: 'no_alternative' | 'failed';
|
|
100
|
+
reason?: string;
|
|
101
|
+
workId?: string;
|
|
102
|
+
scanned?: number;
|
|
103
|
+
skipped?: {
|
|
104
|
+
total: number;
|
|
105
|
+
duplicate: number;
|
|
106
|
+
invalid: number;
|
|
107
|
+
unavailable: number;
|
|
108
|
+
};
|
|
109
|
+
};
|
|
92
110
|
}
|
|
93
111
|
export interface DeliveryProvider {
|
|
94
112
|
deliver(request: DeliveryRequest): Promise<DeliveryResult>;
|
|
@@ -128,6 +128,9 @@ class DownloadManager {
|
|
|
128
128
|
this.novelDownloader = new NovelDownloader_1.NovelDownloader(client, database, fileService, database);
|
|
129
129
|
this.planner = new DownloadPlanner_1.DownloadPlanner(database, {
|
|
130
130
|
deliveredIds: (target, type, ids) => this.deliveryService.deliveredIds(target, type, ids),
|
|
131
|
+
// CANDIDATE SELECTION dedupe: also treats a work whose review submission is
|
|
132
|
+
// still PENDING as taken, so it is skipped instead of submitted again.
|
|
133
|
+
submittedIds: (target, type, ids) => this.deliveryService.submittedIds(target, type, ids),
|
|
131
134
|
// Durable duplicate history handed in by the caller (the batch runner asks
|
|
132
135
|
// the control plane for it). Independent of any local delivery target, so it
|
|
133
136
|
// also works for a shadow run that delivers nowhere.
|
|
@@ -137,7 +140,10 @@ class DownloadManager {
|
|
|
137
140
|
return new Set();
|
|
138
141
|
return new Set(ids.filter((id) => known.has(id)));
|
|
139
142
|
},
|
|
140
|
-
|
|
143
|
+
// Global bounded candidate-scan window (`download.candidateScanLimit`).
|
|
144
|
+
// Per-target `candidateScanLimit` overrides it. Without this the pipeline
|
|
145
|
+
// would only ever see as many candidates as the target's own `limit`.
|
|
146
|
+
}, config.download?.candidateScanLimit);
|
|
141
147
|
this.executor = new DownloadExecutor_1.DownloadExecutor();
|
|
142
148
|
const downloadConfig = config.download ?? {};
|
|
143
149
|
const maxRetries = downloadConfig.maxRetries ?? 3;
|
|
@@ -166,6 +172,11 @@ class DownloadManager {
|
|
|
166
172
|
: config.storage?.databasePath, config.download?.requestDelay ?? 500, this.abortController.signal);
|
|
167
173
|
this.illustrationHandler = new IllustrationTargetHandler_1.IllustrationTargetHandler(client, database, this.rankingService, this.illustrationDownloader, this.pipeline, topicFactory, this.deliveryService);
|
|
168
174
|
this.novelHandler = new NovelTargetHandler_1.NovelTargetHandler(client, database, this.rankingService, this.pipeline, this.novelDownloader, topicFactory, this.deliveryService);
|
|
175
|
+
// The FETCH stage needs the same bound the planner applies to the attempt
|
|
176
|
+
// window, or a `limit: 1` target would ask the ranking API for one work and
|
|
177
|
+
// leave the bounded scan nothing to scan.
|
|
178
|
+
this.illustrationHandler.setDefaultCandidateScanLimit(config.download?.candidateScanLimit);
|
|
179
|
+
this.novelHandler.setDefaultCandidateScanLimit(config.download?.candidateScanLimit);
|
|
169
180
|
}
|
|
170
181
|
setProgressCallback(callback) {
|
|
171
182
|
this.progressReporter.setCallback(callback);
|
|
@@ -18,6 +18,29 @@ export declare class IllustrationTargetHandler {
|
|
|
18
18
|
private readonly deliveryService?;
|
|
19
19
|
/** Outcomes produced during this handle() call (deliveries + terminal non-matches). */
|
|
20
20
|
private outcomes;
|
|
21
|
+
/**
|
|
22
|
+
* Candidate scan of the last pipeline.run() of this handle() call: how many
|
|
23
|
+
* candidates were attempted, which were skipped and why, and whether any
|
|
24
|
+
* job-level outage appeared. This is what turns an empty result into an
|
|
25
|
+
* explicit verdict instead of an ambiguous "completed".
|
|
26
|
+
*/
|
|
27
|
+
private scan;
|
|
28
|
+
/**
|
|
29
|
+
* Global candidate-scan bound (`download.candidateScanLimit`), supplied by
|
|
30
|
+
* DownloadManager. This is what lets the FETCH stage ask for more than one
|
|
31
|
+
* candidate: a scheduled one-post-per-slot target has `limit: 1`, and asking
|
|
32
|
+
* the ranking API for exactly one work is the reason a single duplicate used
|
|
33
|
+
* to be unfixable downstream.
|
|
34
|
+
*/
|
|
35
|
+
private defaultCandidateScanLimit?;
|
|
36
|
+
/** Publish the global candidate-scan bound for the fetch stage. */
|
|
37
|
+
setDefaultCandidateScanLimit(limit: number | undefined): void;
|
|
38
|
+
/**
|
|
39
|
+
* How many candidates to FETCH so the bounded scan has something to choose
|
|
40
|
+
* from. Same rule the planner applies to the attempt window, so fetch and
|
|
41
|
+
* scan agree on one bound.
|
|
42
|
+
*/
|
|
43
|
+
private candidateFetchLimit;
|
|
21
44
|
/**
|
|
22
45
|
* Cell identity for this handle() call. Set only for a single-work cell of a
|
|
23
46
|
* scheduled occurrence; null for ad-hoc runs and N-works-per-run targets, which
|
|
@@ -26,7 +49,15 @@ export declare class IllustrationTargetHandler {
|
|
|
26
49
|
private execution;
|
|
27
50
|
constructor(client: IPixivClient, database: IDatabase, rankingService: RankingService, illustrationDownloader: IllustrationDownloader, pipeline: DownloadPipeline, topicPipelineFactory?: TopicPipelineFactory | undefined, deliveryService?: DeliveryService | undefined);
|
|
28
51
|
handle(target: TargetConfig, execution?: TargetExecutionContext): Promise<TargetOutcome>;
|
|
29
|
-
/**
|
|
52
|
+
/**
|
|
53
|
+
* Reduce the outcomes collected while processing one target to one cell
|
|
54
|
+
* result, including the bounded-scan bookkeeping.
|
|
55
|
+
*
|
|
56
|
+
* A `duplicate` is deliberately NOT a target verdict here: a duplicate is a
|
|
57
|
+
* CANDIDATE problem (skip it and try the next candidate), which is what the
|
|
58
|
+
* scan already did. Returning it as the target outcome is the bug that made a
|
|
59
|
+
* scheduled slot report success after submitting nothing.
|
|
60
|
+
*/
|
|
30
61
|
private summarize;
|
|
31
62
|
private classifyError;
|
|
32
63
|
private fetchIllustrations;
|
|
@@ -56,6 +87,11 @@ export declare class IllustrationTargetHandler {
|
|
|
56
87
|
/**
|
|
57
88
|
* Process ONE candidate work for this cell.
|
|
58
89
|
*
|
|
90
|
+
* Returns what happened to THIS candidate, so the pipeline can advance to the
|
|
91
|
+
* next one when it was unusable. Throwing is reserved for JOB-level failures
|
|
92
|
+
* (dead database / dead token / dead delivery provider): a candidate-level
|
|
93
|
+
* failure is reported as a `skipped` attempt and never as a job verdict.
|
|
94
|
+
*
|
|
59
95
|
* The cell is bound to `illust` BEFORE any side effect: it is the binding, not
|
|
60
96
|
* the candidate list, that decides what a later recovery resumes. The binding
|
|
61
97
|
* is only rolled back when the attempt produced no artifact at all, so in-run
|
|
@@ -67,8 +103,12 @@ export declare class IllustrationTargetHandler {
|
|
|
67
103
|
* delivery mode the DeliveryService creates the durable intent atomically and
|
|
68
104
|
* the result is 'delivery_pending' (NOT submitted — the OutboxWorker confirms
|
|
69
105
|
* the ACK). Persistent/download-only runs are 'stored'.
|
|
106
|
+
*
|
|
107
|
+
* An ALREADY-DELIVERED work is returned as a candidate SKIP, never as a
|
|
108
|
+
* target outcome: the run must try the next candidate instead of ending the
|
|
109
|
+
* slot as a `duplicate`. The delivery idempotency ledger is what makes the
|
|
110
|
+
* second concurrent worker lose this race instead of double-submitting.
|
|
70
111
|
*/
|
|
71
112
|
private recordArtifactOutcome;
|
|
72
|
-
private executionContextFields;
|
|
73
113
|
}
|
|
74
114
|
//# sourceMappingURL=IllustrationTargetHandler.d.ts.map
|