pixivflow 2.19.4 → 2.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/dist/commands/SchedulerCommand.js +36 -0
  2. package/dist/commands/SchedulerRunOnceCommand.d.ts +2 -3
  3. package/dist/commands/SchedulerRunOnceCommand.js +2 -3
  4. package/dist/commands/scheduler-runtime.js +17 -1
  5. package/dist/config/defaults.d.ts +1 -0
  6. package/dist/config/defaults.js +4 -0
  7. package/dist/config/environment.js +19 -0
  8. package/dist/config/types.d.ts +28 -0
  9. package/dist/config/validation.js +25 -0
  10. package/dist/delivery/DeliveryService.d.ts +30 -1
  11. package/dist/delivery/DeliveryService.js +13 -2
  12. package/dist/delivery/HttpMultipartDelivery.js +29 -8
  13. package/dist/delivery/OutboxWorker.d.ts +2 -0
  14. package/dist/delivery/OutboxWorker.js +3 -0
  15. package/dist/delivery/types.d.ts +18 -0
  16. package/dist/download/DownloadManager.js +12 -1
  17. package/dist/download/handlers/IllustrationTargetHandler.d.ts +42 -2
  18. package/dist/download/handlers/IllustrationTargetHandler.js +240 -50
  19. package/dist/download/handlers/NovelTargetHandler.d.ts +47 -0
  20. package/dist/download/handlers/NovelTargetHandler.js +213 -27
  21. package/dist/download/handlers/deliveryContext.d.ts +16 -0
  22. package/dist/download/handlers/deliveryContext.js +32 -0
  23. package/dist/download/pipeline/DownloadPipeline.d.ts +22 -3
  24. package/dist/download/pipeline/DownloadPipeline.js +109 -92
  25. package/dist/download/plan/DownloadPlanner.d.ts +47 -1
  26. package/dist/download/plan/DownloadPlanner.js +105 -15
  27. package/dist/notification/NotificationPolicy.d.ts +12 -0
  28. package/dist/notification/NotificationPolicy.js +86 -0
  29. package/dist/package.json +1 -1
  30. package/dist/scheduler/MultiScheduleManager.js +2 -0
  31. package/dist/scheduler/ScheduleTriggerServer.d.ts +58 -1
  32. package/dist/scheduler/ScheduleTriggerServer.js +207 -40
  33. package/dist/scheduler/SlotCoordinator.d.ts +80 -2
  34. package/dist/scheduler/SlotCoordinator.js +145 -3
  35. package/dist/scheduler/TargetOutcome.d.ts +195 -2
  36. package/dist/scheduler/TargetOutcome.js +202 -3
  37. package/dist/storage/DatabaseMigration.js +11 -0
  38. package/dist/storage/repositories/DeliveryRepository.d.ts +12 -0
  39. package/dist/storage/repositories/DeliveryRepository.js +18 -1
  40. package/dist/storage/repositories/SlotRepository.d.ts +8 -0
  41. package/dist/storage/repositories/SlotRepository.js +6 -2
  42. package/dist/version.js +1 -1
  43. package/dist/webui/package.json +1 -1
  44. package/package.json +1 -1
@@ -157,6 +157,42 @@ class SchedulerCommand extends Command_1.BaseCommand {
157
157
  };
158
158
  },
159
159
  drainOutbox: () => runtime.drainOutbox(),
160
+ refetch: async (targetId, requestId, correlationId) => {
161
+ const cfg = resolveConfig();
162
+ const plans = (cfg.schedules ?? []).filter((plan) => plan.enabled !== false && (0, schedules_1.selectScheduleTargets)(cfg.targets, plan).some((target) => target.id === targetId));
163
+ if (plans.length === 0)
164
+ throw new Error('unknown target');
165
+ if (plans.length !== 1)
166
+ throw new Error('ambiguous target');
167
+ const plan = plans[0];
168
+ const target = (0, schedules_1.selectScheduleTargets)(cfg.targets, plan).find((item) => item.id === targetId);
169
+ const now = new Date();
170
+ const date = new Intl.DateTimeFormat('en-CA', {
171
+ timeZone: plan.timezone ?? 'UTC', year: 'numeric', month: '2-digit', day: '2-digit',
172
+ }).format(now);
173
+ const slot = {
174
+ slotId: `${plan.id}@manual-${requestId.toLowerCase()}`,
175
+ scheduleId: plan.id,
176
+ occurrenceAt: now.getTime(),
177
+ occurrenceDate: date,
178
+ occurrenceLabel: 'manual',
179
+ timezone: plan.timezone ?? 'UTC',
180
+ triggerSource: 'manual',
181
+ slotName: '审核群重抓',
182
+ slotDate: date,
183
+ manualRequestId: requestId,
184
+ correlationId: correlationId || undefined,
185
+ };
186
+ const existing = runtime.database.slots.getSlot(slot.slotId);
187
+ if (existing && (existing.scheduleId !== plan.id || existing.targetIds.length !== 1 || existing.targetIds[0] !== targetId)) {
188
+ throw new Error('ambiguous target');
189
+ }
190
+ const prepared = coordinator.prepare(slot, plan, [target]);
191
+ if (prepared.alreadyCompleted)
192
+ return { slotId: slot.slotId, disposition: 'already_completed' };
193
+ const started = manager.triggerSchedule(plan.id, { slot, onlyTarget: targetId, triggerSource: 'manual' });
194
+ return { slotId: slot.slotId, disposition: started ? 'accepted' : 'queued' };
195
+ },
160
196
  status: (scheduleId) => {
161
197
  const cfg = resolveConfig();
162
198
  const plan = findPlan(cfg, scheduleId);
@@ -2,9 +2,8 @@
2
2
  * Scheduler run-once command
3
3
  *
4
4
  * Runs every enabled schedule's download plan exactly once, then exits.
5
- * This is the backend for the review-group "重抓/换一张" refetch button:
6
- * a second scheduler daemon would never exit (and would double-fire cron),
7
- * so refetch needs a bounded one-shot invocation instead.
5
+ * This is the local CLI path. Remote review-group refetch uses the authenticated
6
+ * trigger server and a durable manual Slot so a sleeping worker can recover it.
8
7
  */
9
8
  import { BaseCommand } from './Command';
10
9
  import { CommandCategory } from './metadata';
@@ -3,9 +3,8 @@
3
3
  * Scheduler run-once command
4
4
  *
5
5
  * Runs every enabled schedule's download plan exactly once, then exits.
6
- * This is the backend for the review-group "重抓/换一张" refetch button:
7
- * a second scheduler daemon would never exit (and would double-fire cron),
8
- * so refetch needs a bounded one-shot invocation instead.
6
+ * This is the local CLI path. Remote review-group refetch uses the authenticated
7
+ * trigger server and a durable manual Slot so a sleeping worker can recover it.
9
8
  */
10
9
  Object.defineProperty(exports, "__esModule", { value: true });
11
10
  exports.SchedulerRunOnceCommand = void 0;
@@ -438,10 +438,26 @@ async function createSchedulerRuntime(configPathArg) {
438
438
  if (!target.id)
439
439
  return;
440
440
  options.onTargetOutcome?.(target.id, outcome);
441
- if (!scheduleSlot)
441
+ if (!scheduleSlot) {
442
+ // No durable slot (run-once CLI): nothing to converge or report.
442
443
  return;
444
+ }
443
445
  coordinator.applyOutcome(scheduleSlot.slotId, target.id, outcome);
444
446
  notificationPolicy.noteOutcome(scheduleSlot.slotId, scheduleSlot, schedule, target, outcome);
447
+ // A remote manual replacement ("重抓") must report its terminal verdict
448
+ // back to the reviewer. Only terminal outcomes are reported: a candidate
449
+ // the scan skipped is not a verdict, and a durable delivery intent
450
+ // ('delivery_pending' / later 'submitted') is reported through the
451
+ // replacement submission itself (the caller correlates on requestId).
452
+ const manualRequestId = scheduleSlot.manualRequestId;
453
+ if (manualRequestId) {
454
+ const terminal = outcome.kind === 'no_candidate' ||
455
+ outcome.kind === 'duplicate' ||
456
+ (outcome.kind === 'failed' && !outcome.retryable);
457
+ if (terminal) {
458
+ notificationPolicy.noteRefetchOutcome(scheduleSlot, schedule, target, manualRequestId, outcome);
459
+ }
460
+ }
445
461
  });
446
462
  if (scheduleSlot) {
447
463
  downloadManager.slotContext = {
@@ -38,6 +38,7 @@ export declare const DEFAULT_CONFIG: {
38
38
  readonly maxRetries: 3;
39
39
  readonly retryDelay: 2000;
40
40
  readonly timeout: 60000;
41
+ readonly candidateScanLimit: 5;
41
42
  };
42
43
  readonly initialDelay: 0;
43
44
  };
@@ -41,6 +41,10 @@ exports.DEFAULT_CONFIG = {
41
41
  maxRetries: 3,
42
42
  retryDelay: 2000,
43
43
  timeout: 60000,
44
+ // Candidates one execution may ATTEMPT while looking for an eligible work.
45
+ // Bounded so a ranking page full of already-delivered works cannot loop,
46
+ // but large enough that a single duplicate no longer burns the slot.
47
+ candidateScanLimit: 5,
44
48
  },
45
49
  initialDelay: 0,
46
50
  };
@@ -101,6 +101,25 @@ function applyEnvironmentOverrides(config) {
101
101
  overridden.scheduler.enabled = process.env.PIXIV_SCHEDULER_ENABLED.toLowerCase() === 'true';
102
102
  }
103
103
  }
104
+ // Override the bounded candidate-scan window. A non-numeric or non-positive
105
+ // value is ignored (the config-file value stays authoritative) rather than
106
+ // silently becoming NaN, which would disable the bound entirely.
107
+ if (process.env.PIXIV_CANDIDATE_SCAN_LIMIT !== undefined) {
108
+ const parsed = Number.parseInt(process.env.PIXIV_CANDIDATE_SCAN_LIMIT, 10);
109
+ if (Number.isFinite(parsed) && parsed > 0) {
110
+ if (!overridden.download) {
111
+ overridden.download = { candidateScanLimit: parsed };
112
+ }
113
+ else {
114
+ overridden.download.candidateScanLimit = parsed;
115
+ }
116
+ }
117
+ else {
118
+ logger_1.logger.warn('Ignoring PIXIV_CANDIDATE_SCAN_LIMIT: expected a positive integer', {
119
+ value: process.env.PIXIV_CANDIDATE_SCAN_LIMIT,
120
+ });
121
+ }
122
+ }
104
123
  // Override proxy from environment variables
105
124
  // Priority: all_proxy > https_proxy > http_proxy
106
125
  const proxyUrl = process.env.all_proxy || process.env.ALL_PROXY ||
@@ -81,6 +81,20 @@ export interface TargetConfig {
81
81
  * Maximum works to download per execution for this tag.
82
82
  */
83
83
  limit?: number;
84
+ /**
85
+ * Maximum CANDIDATE works one execution may attempt (not produce) while
86
+ * looking for an eligible work. A scheduled "one post per slot" target has
87
+ * `limit: 1`, so without this the run would hold a single candidate: if that
88
+ * one candidate is already delivered / deleted / private, the slot ends with
89
+ * nothing submitted. The scan walks up to N candidates in ranking order,
90
+ * skipping unusable ones, and is strictly bounded so a page full of
91
+ * duplicates cannot loop.
92
+ *
93
+ * Overrides `download.candidateScanLimit` / `PIXIV_CANDIDATE_SCAN_LIMIT`.
94
+ * Clamped to 1..100 and never below `limit`, so a multi-work target can still
95
+ * fill its own limit.
96
+ */
97
+ candidateScanLimit?: number;
84
98
  /**
85
99
  * Search target parameter for Pixiv API.
86
100
  */
@@ -503,6 +517,13 @@ export interface HttpMultipartDeliveryConfig {
503
517
  url: string;
504
518
  /** Optional JSON endpoint used for no-match operational notifications. */
505
519
  notificationUrl?: string;
520
+ /**
521
+ * Optional JSON endpoint for remote manual replacement ("重抓") terminal
522
+ * verdicts (no_alternative / failed). Must be http(s); if unset, manual
523
+ * outcome reports are skipped (the work itself is still durable). Reuses
524
+ * `headers` for auth, so no extra credential is needed.
525
+ */
526
+ refetchOutcomeUrl?: string;
506
527
  method?: 'POST' | 'PUT';
507
528
  /** 支持 ${ENV_NAME} 环境变量插值 */
508
529
  headers?: Record<string, string>;
@@ -655,6 +676,13 @@ export interface StandaloneConfig {
655
676
  * Default: 60000
656
677
  */
657
678
  timeout?: number;
679
+ /**
680
+ * How many CANDIDATE works one execution may attempt while looking for an
681
+ * eligible one. Per-target `candidateScanLimit` overrides this.
682
+ *
683
+ * Default: 5 (clamped to 1..100)
684
+ */
685
+ candidateScanLimit?: number;
658
686
  };
659
687
  }
660
688
  //# sourceMappingURL=types.d.ts.map
@@ -302,6 +302,16 @@ function validateConfig(config, location, databasePath) {
302
302
  errors.push(`${prefix}.notificationUrl: Must be a valid HTTP or HTTPS URL`);
303
303
  }
304
304
  }
305
+ if (delivery.refetchOutcomeUrl && !/\$\{[A-Za-z_][A-Za-z0-9_]*\}/.test(delivery.refetchOutcomeUrl)) {
306
+ try {
307
+ const url = new URL(delivery.refetchOutcomeUrl);
308
+ if (!['http:', 'https:'].includes(url.protocol))
309
+ throw new Error('unsupported protocol');
310
+ }
311
+ catch {
312
+ errors.push(`${prefix}.refetchOutcomeUrl: Must be a valid HTTP or HTTPS URL`);
313
+ }
314
+ }
305
315
  if (delivery.readinessUrl && !/\$\{[A-Za-z_][A-Za-z0-9_]*\}/.test(delivery.readinessUrl)) {
306
316
  try {
307
317
  const url = new URL(delivery.readinessUrl);
@@ -469,6 +479,21 @@ function validateConfig(config, location, databasePath) {
469
479
  if (config.download.maxRetries !== undefined && (config.download.maxRetries < 0 || config.download.maxRetries > 10)) {
470
480
  warnings.push('download.maxRetries: Should be between 0 and 10');
471
481
  }
482
+ if (config.download.candidateScanLimit !== undefined &&
483
+ (!Number.isInteger(config.download.candidateScanLimit) ||
484
+ config.download.candidateScanLimit < 1 ||
485
+ config.download.candidateScanLimit > 100)) {
486
+ warnings.push('download.candidateScanLimit: Should be an integer between 1 and 100');
487
+ }
488
+ }
489
+ // The bounded candidate scan is what stops a page full of duplicates from
490
+ // burning a whole scheduled slot, so a non-integer value is reported here.
491
+ for (const target of config.targets ?? []) {
492
+ if (target.candidateScanLimit === undefined)
493
+ continue;
494
+ if (!Number.isInteger(target.candidateScanLimit) || target.candidateScanLimit < 1 || target.candidateScanLimit > 100) {
495
+ warnings.push(`targets.${target.id ?? target.tag ?? '?'}.candidateScanLimit: Should be an integer between 1 and 100`);
496
+ }
472
497
  }
473
498
  // Validate log level
474
499
  if (config.logLevel && !['debug', 'info', 'warn', 'error'].includes(config.logLevel)) {
@@ -19,6 +19,26 @@ export interface EnqueueResult {
19
19
  /** True when this call created a brand new delivery row. */
20
20
  created: boolean;
21
21
  }
22
+ /**
23
+ * Machine-readable terminal verdict of a remote manual replacement ("重抓"),
24
+ * reported back to the requester (TelePost) through the durable outbox.
25
+ * Never contains credentials; ids are the opaque request UUID and Pixiv work ids.
26
+ */
27
+ export interface RefetchOutcomePayload {
28
+ requestId: string;
29
+ /** 'no_alternative' | 'failed' (replacement success rides the submission). */
30
+ disposition: 'no_alternative' | 'failed';
31
+ reason?: string;
32
+ workId?: string;
33
+ /** Bounded candidate-scan bookkeeping for diagnostics (spec-compatible). */
34
+ scanned?: number;
35
+ skipped?: {
36
+ total: number;
37
+ duplicate: number;
38
+ invalid: number;
39
+ unavailable: number;
40
+ };
41
+ }
22
42
  export declare class DeliveryService {
23
43
  private readonly database;
24
44
  constructor(database: Database);
@@ -35,6 +55,15 @@ export declare class DeliveryService {
35
55
  isAlreadyDelivered(deliveryTarget: string, workType: string, pixivId: string): boolean;
36
56
  /** Batch form for pre-lock candidate filtering. */
37
57
  deliveredIds(deliveryTarget: string, workType: string, pixivIds: string[]): Set<string>;
58
+ /**
59
+ * Bulk CANDIDATE SELECTION dedupe: works already delivered to this target, or
60
+ * whose review submission is still PENDING an answer. A pending work is
61
+ * already submitted for review, so it must not be selected (and submitted)
62
+ * again; the scan moves on to the next candidate instead. Within-slot RESUME
63
+ * keeps using `isAlreadyDelivered` — a cell continuing its OWN pending work is
64
+ * resuming it, not duplicating it.
65
+ */
66
+ submittedIds(deliveryTarget: string, workType: string, pixivIds: string[]): Set<string>;
38
67
  /**
39
68
  * Atomically create the delivery intent + outbox row and advance the cell.
40
69
  * Missing local artifact files are treated as a recoverable error (the
@@ -67,7 +96,7 @@ export declare class DeliveryService {
67
96
  created: boolean;
68
97
  };
69
98
  /** Enqueue a durable notification (retried independently; never affects content). */
70
- enqueueNotification(deliveryTarget: string, text: string, idempotencyKey: string): void;
99
+ enqueueNotification(deliveryTarget: string, text: string, idempotencyKey: string, refetchOutcome?: RefetchOutcomePayload): void;
71
100
  private guardCell;
72
101
  private contextFrom;
73
102
  }
@@ -33,6 +33,17 @@ class DeliveryService {
33
33
  deliveredIds(deliveryTarget, workType, pixivIds) {
34
34
  return this.database.deliveries.deliveredIds(deliveryTarget, workType, pixivIds);
35
35
  }
36
+ /**
37
+ * Bulk CANDIDATE SELECTION dedupe: works already delivered to this target, or
38
+ * whose review submission is still PENDING an answer. A pending work is
39
+ * already submitted for review, so it must not be selected (and submitted)
40
+ * again; the scan moves on to the next candidate instead. Within-slot RESUME
41
+ * keeps using `isAlreadyDelivered` — a cell continuing its OWN pending work is
42
+ * resuming it, not duplicating it.
43
+ */
44
+ submittedIds(deliveryTarget, workType, pixivIds) {
45
+ return this.database.deliveries.submittedIds(deliveryTarget, workType, pixivIds);
46
+ }
36
47
  /**
37
48
  * Atomically create the delivery intent + outbox row and advance the cell.
38
49
  * Missing local artifact files are treated as a recoverable error (the
@@ -164,12 +175,12 @@ class DeliveryService {
164
175
  return { deliveryId: row.id, created };
165
176
  }
166
177
  /** Enqueue a durable notification (retried independently; never affects content). */
167
- enqueueNotification(deliveryTarget, text, idempotencyKey) {
178
+ enqueueNotification(deliveryTarget, text, idempotencyKey, refetchOutcome) {
168
179
  this.database.outbox.enqueue({
169
180
  kind: 'notification',
170
181
  deliveryTarget,
171
182
  idempotencyKey,
172
- payload: { text },
183
+ payload: refetchOutcome ? { text, refetchOutcome } : { text },
173
184
  });
174
185
  }
175
186
  guardCell(slotId, targetId, next) {
@@ -133,33 +133,54 @@ class HttpMultipartDelivery {
133
133
  }
134
134
  /** Single notification attempt; the outbox owns retries. */
135
135
  async notifyOnce(request) {
136
- const url = this.config.notificationUrl?.trim();
137
- if (!url)
138
- throw new Error('HTTP delivery notificationUrl is not configured');
136
+ const outcome = request.refetchOutcome;
137
+ const url = (outcome ? this.config.refetchOutcomeUrl?.trim() : this.config.notificationUrl?.trim());
138
+ if (!url) {
139
+ throw new Error(outcome
140
+ ? 'HTTP delivery refetchOutcomeUrl is not configured'
141
+ : 'HTTP delivery notificationUrl is not configured');
142
+ }
139
143
  const headers = {
140
144
  ...this.resolveHeaders(this.config.headers ?? {}),
141
145
  'Content-Type': 'application/json',
142
146
  };
147
+ // Refetch verdicts are machine-readable JSON (the requester's review state
148
+ // machine consumes disposition, not prose). Plain notifications remain
149
+ // {text, idempotency_key}.
150
+ const body = outcome
151
+ ? {
152
+ request_id: outcome.requestId,
153
+ disposition: outcome.disposition,
154
+ reason: outcome.reason,
155
+ work_id: outcome.workId,
156
+ scanned: outcome.scanned,
157
+ skipped: outcome.skipped,
158
+ }
159
+ : { text: request.text, idempotency_key: request.idempotencyKey };
143
160
  const options = {
144
161
  method: 'POST',
145
162
  headers,
146
- body: JSON.stringify({ text: request.text, idempotency_key: request.idempotencyKey }),
163
+ body: JSON.stringify(body),
147
164
  };
148
165
  if (this.dispatcher)
149
166
  options.dispatcher = this.dispatcher;
150
167
  const response = await fetch(this.interpolateEnvironment(url), options);
151
168
  const text = await response.text();
152
- let body = text;
169
+ let parsed = text;
153
170
  if (text) {
154
171
  try {
155
- body = JSON.parse(text);
172
+ parsed = JSON.parse(text);
156
173
  }
157
174
  catch { /* plain ok */ }
158
175
  }
159
176
  if (!response.ok)
160
177
  throw new Error(`notification endpoint returned HTTP ${response.status}`);
161
- logger_1.logger.info('HTTP delivery notification sent', { url: (0, redact_1.redactUrl)(url), status: response.status });
162
- return { status: response.status, body };
178
+ logger_1.logger.info('HTTP delivery notification sent', {
179
+ url: (0, redact_1.redactUrl)(url),
180
+ status: response.status,
181
+ hasRefetchOutcome: Boolean(outcome),
182
+ });
183
+ return { status: response.status, body: parsed };
163
184
  }
164
185
  async attempt(request) {
165
186
  const fields = this.resolveFields({ ...(this.config.fields ?? {}), ...(request.fields ?? {}) }, request);
@@ -29,6 +29,8 @@ export interface DeliveryPayload {
29
29
  /** Notification side-effect payload. */
30
30
  export interface NotificationPayload {
31
31
  text: string;
32
+ /** Optional structured remote-manual-replacement verdict (refetch outcome). */
33
+ refetchOutcome?: unknown;
32
34
  }
33
35
  /** Exponential backoff with jitter, capped. */
34
36
  export declare function backoffDelayMs(attempt: number, base: number, max: number): number;
@@ -211,6 +211,9 @@ class OutboxWorker {
211
211
  await this.dispatcher.notify(row.deliveryTarget, {
212
212
  text: payload.text,
213
213
  idempotencyKey: row.idempotencyKey ?? row.id,
214
+ refetchOutcome: payload.refetchOutcome !== undefined
215
+ ? payload.refetchOutcome
216
+ : undefined,
214
217
  });
215
218
  }
216
219
  else {
@@ -89,6 +89,24 @@ export interface DeliveryResult {
89
89
  export interface DeliveryNotificationRequest {
90
90
  text: string;
91
91
  idempotencyKey: string;
92
+ /**
93
+ * Optional structured remote-manual-replacement verdict. When present the
94
+ * delivery sends a JSON body to the target's `refetchOutcomeUrl` instead of
95
+ * a plain text notification to `notificationUrl` (auth reuses `headers`).
96
+ */
97
+ refetchOutcome?: {
98
+ requestId: string;
99
+ disposition: 'no_alternative' | 'failed';
100
+ reason?: string;
101
+ workId?: string;
102
+ scanned?: number;
103
+ skipped?: {
104
+ total: number;
105
+ duplicate: number;
106
+ invalid: number;
107
+ unavailable: number;
108
+ };
109
+ };
92
110
  }
93
111
  export interface DeliveryProvider {
94
112
  deliver(request: DeliveryRequest): Promise<DeliveryResult>;
@@ -128,6 +128,9 @@ class DownloadManager {
128
128
  this.novelDownloader = new NovelDownloader_1.NovelDownloader(client, database, fileService, database);
129
129
  this.planner = new DownloadPlanner_1.DownloadPlanner(database, {
130
130
  deliveredIds: (target, type, ids) => this.deliveryService.deliveredIds(target, type, ids),
131
+ // CANDIDATE SELECTION dedupe: also treats a work whose review submission is
132
+ // still PENDING as taken, so it is skipped instead of submitted again.
133
+ submittedIds: (target, type, ids) => this.deliveryService.submittedIds(target, type, ids),
131
134
  // Durable duplicate history handed in by the caller (the batch runner asks
132
135
  // the control plane for it). Independent of any local delivery target, so it
133
136
  // also works for a shadow run that delivers nowhere.
@@ -137,7 +140,10 @@ class DownloadManager {
137
140
  return new Set();
138
141
  return new Set(ids.filter((id) => known.has(id)));
139
142
  },
140
- });
143
+ // Global bounded candidate-scan window (`download.candidateScanLimit`).
144
+ // Per-target `candidateScanLimit` overrides it. Without this the pipeline
145
+ // would only ever see as many candidates as the target's own `limit`.
146
+ }, config.download?.candidateScanLimit);
141
147
  this.executor = new DownloadExecutor_1.DownloadExecutor();
142
148
  const downloadConfig = config.download ?? {};
143
149
  const maxRetries = downloadConfig.maxRetries ?? 3;
@@ -166,6 +172,11 @@ class DownloadManager {
166
172
  : config.storage?.databasePath, config.download?.requestDelay ?? 500, this.abortController.signal);
167
173
  this.illustrationHandler = new IllustrationTargetHandler_1.IllustrationTargetHandler(client, database, this.rankingService, this.illustrationDownloader, this.pipeline, topicFactory, this.deliveryService);
168
174
  this.novelHandler = new NovelTargetHandler_1.NovelTargetHandler(client, database, this.rankingService, this.pipeline, this.novelDownloader, topicFactory, this.deliveryService);
175
+ // The FETCH stage needs the same bound the planner applies to the attempt
176
+ // window, or a `limit: 1` target would ask the ranking API for one work and
177
+ // leave the bounded scan nothing to scan.
178
+ this.illustrationHandler.setDefaultCandidateScanLimit(config.download?.candidateScanLimit);
179
+ this.novelHandler.setDefaultCandidateScanLimit(config.download?.candidateScanLimit);
169
180
  }
170
181
  setProgressCallback(callback) {
171
182
  this.progressReporter.setCallback(callback);
@@ -18,6 +18,29 @@ export declare class IllustrationTargetHandler {
18
18
  private readonly deliveryService?;
19
19
  /** Outcomes produced during this handle() call (deliveries + terminal non-matches). */
20
20
  private outcomes;
21
+ /**
22
+ * Candidate scan of the last pipeline.run() of this handle() call: how many
23
+ * candidates were attempted, which were skipped and why, and whether any
24
+ * job-level outage appeared. This is what turns an empty result into an
25
+ * explicit verdict instead of an ambiguous "completed".
26
+ */
27
+ private scan;
28
+ /**
29
+ * Global candidate-scan bound (`download.candidateScanLimit`), supplied by
30
+ * DownloadManager. This is what lets the FETCH stage ask for more than one
31
+ * candidate: a scheduled one-post-per-slot target has `limit: 1`, and asking
32
+ * the ranking API for exactly one work is the reason a single duplicate used
33
+ * to be unfixable downstream.
34
+ */
35
+ private defaultCandidateScanLimit?;
36
+ /** Publish the global candidate-scan bound for the fetch stage. */
37
+ setDefaultCandidateScanLimit(limit: number | undefined): void;
38
+ /**
39
+ * How many candidates to FETCH so the bounded scan has something to choose
40
+ * from. Same rule the planner applies to the attempt window, so fetch and
41
+ * scan agree on one bound.
42
+ */
43
+ private candidateFetchLimit;
21
44
  /**
22
45
  * Cell identity for this handle() call. Set only for a single-work cell of a
23
46
  * scheduled occurrence; null for ad-hoc runs and N-works-per-run targets, which
@@ -26,7 +49,15 @@ export declare class IllustrationTargetHandler {
26
49
  private execution;
27
50
  constructor(client: IPixivClient, database: IDatabase, rankingService: RankingService, illustrationDownloader: IllustrationDownloader, pipeline: DownloadPipeline, topicPipelineFactory?: TopicPipelineFactory | undefined, deliveryService?: DeliveryService | undefined);
28
51
  handle(target: TargetConfig, execution?: TargetExecutionContext): Promise<TargetOutcome>;
29
- /** Reduce the outcomes collected while processing one target to one cell result. */
52
+ /**
53
+ * Reduce the outcomes collected while processing one target to one cell
54
+ * result, including the bounded-scan bookkeeping.
55
+ *
56
+ * A `duplicate` is deliberately NOT a target verdict here: a duplicate is a
57
+ * CANDIDATE problem (skip it and try the next candidate), which is what the
58
+ * scan already did. Returning it as the target outcome is the bug that made a
59
+ * scheduled slot report success after submitting nothing.
60
+ */
30
61
  private summarize;
31
62
  private classifyError;
32
63
  private fetchIllustrations;
@@ -56,6 +87,11 @@ export declare class IllustrationTargetHandler {
56
87
  /**
57
88
  * Process ONE candidate work for this cell.
58
89
  *
90
+ * Returns what happened to THIS candidate, so the pipeline can advance to the
91
+ * next one when it was unusable. Throwing is reserved for JOB-level failures
92
+ * (dead database / dead token / dead delivery provider): a candidate-level
93
+ * failure is reported as a `skipped` attempt and never as a job verdict.
94
+ *
59
95
  * The cell is bound to `illust` BEFORE any side effect: it is the binding, not
60
96
  * the candidate list, that decides what a later recovery resumes. The binding
61
97
  * is only rolled back when the attempt produced no artifact at all, so in-run
@@ -67,8 +103,12 @@ export declare class IllustrationTargetHandler {
67
103
  * delivery mode the DeliveryService creates the durable intent atomically and
68
104
  * the result is 'delivery_pending' (NOT submitted — the OutboxWorker confirms
69
105
  * the ACK). Persistent/download-only runs are 'stored'.
106
+ *
107
+ * An ALREADY-DELIVERED work is returned as a candidate SKIP, never as a
108
+ * target outcome: the run must try the next candidate instead of ending the
109
+ * slot as a `duplicate`. The delivery idempotency ledger is what makes the
110
+ * second concurrent worker lose this race instead of double-submitting.
70
111
  */
71
112
  private recordArtifactOutcome;
72
- private executionContextFields;
73
113
  }
74
114
  //# sourceMappingURL=IllustrationTargetHandler.d.ts.map