pixivflow 2.19.4 → 2.19.5

package/dist/config/defaults.d.ts

@@ -38,6 +38,7 @@ export declare const DEFAULT_CONFIG: {
         readonly maxRetries: 3;
         readonly retryDelay: 2000;
         readonly timeout: 60000;
+        readonly candidateScanLimit: 5;
     };
     readonly initialDelay: 0;
 };

package/dist/config/defaults.js

@@ -41,6 +41,10 @@ exports.DEFAULT_CONFIG = {
         maxRetries: 3,
         retryDelay: 2000,
         timeout: 60000,
+        // Candidates one execution may ATTEMPT while looking for an eligible work.
+        // Bounded so a ranking page full of already-delivered works cannot loop,
+        // but large enough that a single duplicate no longer burns the slot.
+        candidateScanLimit: 5,
     },
     initialDelay: 0,
 };

package/dist/config/environment.js

@@ -101,6 +101,25 @@ function applyEnvironmentOverrides(config) {
             overridden.scheduler.enabled = process.env.PIXIV_SCHEDULER_ENABLED.toLowerCase() === 'true';
         }
     }
+    // Override the bounded candidate-scan window. A non-numeric or non-positive
+    // value is ignored (the config-file value stays authoritative) rather than
+    // silently becoming NaN, which would disable the bound entirely.
+    if (process.env.PIXIV_CANDIDATE_SCAN_LIMIT !== undefined) {
+        const parsed = Number.parseInt(process.env.PIXIV_CANDIDATE_SCAN_LIMIT, 10);
+        if (Number.isFinite(parsed) && parsed > 0) {
+            if (!overridden.download) {
+                overridden.download = { candidateScanLimit: parsed };
+            }
+            else {
+                overridden.download.candidateScanLimit = parsed;
+            }
+        }
+        else {
+            logger_1.logger.warn('Ignoring PIXIV_CANDIDATE_SCAN_LIMIT: expected a positive integer', {
+                value: process.env.PIXIV_CANDIDATE_SCAN_LIMIT,
+            });
+        }
+    }
     // Override proxy from environment variables
     // Priority: all_proxy > https_proxy > http_proxy
     const proxyUrl = process.env.all_proxy || process.env.ALL_PROXY ||

package/dist/config/types.d.ts

@@ -81,6 +81,20 @@ export interface TargetConfig {
      * Maximum works to download per execution for this tag.
      */
     limit?: number;
+    /**
+     * Maximum CANDIDATE works one execution may attempt (not produce) while
+     * looking for an eligible work. A scheduled "one post per slot" target has
+     * `limit: 1`, so without this the run would hold a single candidate: if that
+     * one candidate is already delivered / deleted / private, the slot ends with
+     * nothing submitted. The scan walks up to N candidates in ranking order,
+     * skipping unusable ones, and is strictly bounded so a page full of
+     * duplicates cannot loop.
+     *
+     * Overrides `download.candidateScanLimit` / `PIXIV_CANDIDATE_SCAN_LIMIT`.
+     * Clamped to 1..100 and never below `limit`, so a multi-work target can still
+     * fill its own limit.
+     */
+    candidateScanLimit?: number;
     /**
      * Search target parameter for Pixiv API.
      */
@@ -655,6 +669,13 @@ export interface StandaloneConfig {
          * Default: 60000
          */
         timeout?: number;
+        /**
+         * How many CANDIDATE works one execution may attempt while looking for an
+         * eligible one. Per-target `candidateScanLimit` overrides this.
+         *
+         * Default: 5 (clamped to 1..100)
+         */
+        candidateScanLimit?: number;
     };
 }
 //# sourceMappingURL=types.d.ts.map

package/dist/config/validation.js

@@ -469,6 +469,21 @@ function validateConfig(config, location, databasePath) {
         if (config.download.maxRetries !== undefined && (config.download.maxRetries < 0 || config.download.maxRetries > 10)) {
             warnings.push('download.maxRetries: Should be between 0 and 10');
         }
+        if (config.download.candidateScanLimit !== undefined &&
+            (!Number.isInteger(config.download.candidateScanLimit) ||
+                config.download.candidateScanLimit < 1 ||
+                config.download.candidateScanLimit > 100)) {
+            warnings.push('download.candidateScanLimit: Should be an integer between 1 and 100');
+        }
+    }
+    // The bounded candidate scan is what stops a page full of duplicates from
+    // burning a whole scheduled slot, so a non-integer value is reported here.
+    for (const target of config.targets ?? []) {
+        if (target.candidateScanLimit === undefined)
+            continue;
+        if (!Number.isInteger(target.candidateScanLimit) || target.candidateScanLimit < 1 || target.candidateScanLimit > 100) {
+            warnings.push(`targets.${target.id ?? target.tag ?? '?'}.candidateScanLimit: Should be an integer between 1 and 100`);
+        }
     }
     // Validate log level
     if (config.logLevel && !['debug', 'info', 'warn', 'error'].includes(config.logLevel)) {

package/dist/delivery/DeliveryService.d.ts

@@ -35,6 +35,15 @@ export declare class DeliveryService {
     isAlreadyDelivered(deliveryTarget: string, workType: string, pixivId: string): boolean;
     /** Batch form for pre-lock candidate filtering. */
     deliveredIds(deliveryTarget: string, workType: string, pixivIds: string[]): Set<string>;
+    /**
+     * Bulk CANDIDATE SELECTION dedupe: works already delivered to this target, or
+     * whose review submission is still PENDING an answer. A pending work is
+     * already submitted for review, so it must not be selected (and submitted)
+     * again; the scan moves on to the next candidate instead. Within-slot RESUME
+     * keeps using `isAlreadyDelivered` — a cell continuing its OWN pending work is
+     * resuming it, not duplicating it.
+     */
+    submittedIds(deliveryTarget: string, workType: string, pixivIds: string[]): Set<string>;
     /**
      * Atomically create the delivery intent + outbox row and advance the cell.
      * Missing local artifact files are treated as a recoverable error (the

package/dist/delivery/DeliveryService.js

@@ -33,6 +33,17 @@ class DeliveryService {
     deliveredIds(deliveryTarget, workType, pixivIds) {
         return this.database.deliveries.deliveredIds(deliveryTarget, workType, pixivIds);
     }
+    /**
+     * Bulk CANDIDATE SELECTION dedupe: works already delivered to this target, or
+     * whose review submission is still PENDING an answer. A pending work is
+     * already submitted for review, so it must not be selected (and submitted)
+     * again; the scan moves on to the next candidate instead. Within-slot RESUME
+     * keeps using `isAlreadyDelivered` — a cell continuing its OWN pending work is
+     * resuming it, not duplicating it.
+     */
+    submittedIds(deliveryTarget, workType, pixivIds) {
+        return this.database.deliveries.submittedIds(deliveryTarget, workType, pixivIds);
+    }
     /**
      * Atomically create the delivery intent + outbox row and advance the cell.
      * Missing local artifact files are treated as a recoverable error (the

package/dist/download/DownloadManager.js

@@ -128,6 +128,9 @@ class DownloadManager {
         this.novelDownloader = new NovelDownloader_1.NovelDownloader(client, database, fileService, database);
         this.planner = new DownloadPlanner_1.DownloadPlanner(database, {
             deliveredIds: (target, type, ids) => this.deliveryService.deliveredIds(target, type, ids),
+            // CANDIDATE SELECTION dedupe: also treats a work whose review submission is
+            // still PENDING as taken, so it is skipped instead of submitted again.
+            submittedIds: (target, type, ids) => this.deliveryService.submittedIds(target, type, ids),
             // Durable duplicate history handed in by the caller (the batch runner asks
             // the control plane for it). Independent of any local delivery target, so it
             // also works for a shadow run that delivers nowhere.
@@ -137,7 +140,10 @@ class DownloadManager {
                     return new Set();
                 return new Set(ids.filter((id) => known.has(id)));
             },
-        });
+            // Global bounded candidate-scan window (`download.candidateScanLimit`).
+            // Per-target `candidateScanLimit` overrides it. Without this the pipeline
+            // would only ever see as many candidates as the target's own `limit`.
+        }, config.download?.candidateScanLimit);
         this.executor = new DownloadExecutor_1.DownloadExecutor();
         const downloadConfig = config.download ?? {};
         const maxRetries = downloadConfig.maxRetries ?? 3;
@@ -166,6 +172,11 @@ class DownloadManager {
             : config.storage?.databasePath, config.download?.requestDelay ?? 500, this.abortController.signal);
         this.illustrationHandler = new IllustrationTargetHandler_1.IllustrationTargetHandler(client, database, this.rankingService, this.illustrationDownloader, this.pipeline, topicFactory, this.deliveryService);
         this.novelHandler = new NovelTargetHandler_1.NovelTargetHandler(client, database, this.rankingService, this.pipeline, this.novelDownloader, topicFactory, this.deliveryService);
+        // The FETCH stage needs the same bound the planner applies to the attempt
+        // window, or a `limit: 1` target would ask the ranking API for one work and
+        // leave the bounded scan nothing to scan.
+        this.illustrationHandler.setDefaultCandidateScanLimit(config.download?.candidateScanLimit);
+        this.novelHandler.setDefaultCandidateScanLimit(config.download?.candidateScanLimit);
     }
     setProgressCallback(callback) {
         this.progressReporter.setCallback(callback);

package/dist/download/handlers/IllustrationTargetHandler.d.ts

@@ -18,6 +18,29 @@ export declare class IllustrationTargetHandler {
     private readonly deliveryService?;
     /** Outcomes produced during this handle() call (deliveries + terminal non-matches). */
     private outcomes;
+    /**
+     * Candidate scan of the last pipeline.run() of this handle() call: how many
+     * candidates were attempted, which were skipped and why, and whether any
+     * job-level outage appeared. This is what turns an empty result into an
+     * explicit verdict instead of an ambiguous "completed".
+     */
+    private scan;
+    /**
+     * Global candidate-scan bound (`download.candidateScanLimit`), supplied by
+     * DownloadManager. This is what lets the FETCH stage ask for more than one
+     * candidate: a scheduled one-post-per-slot target has `limit: 1`, and asking
+     * the ranking API for exactly one work is the reason a single duplicate used
+     * to be unfixable downstream.
+     */
+    private defaultCandidateScanLimit?;
+    /** Publish the global candidate-scan bound for the fetch stage. */
+    setDefaultCandidateScanLimit(limit: number | undefined): void;
+    /**
+     * How many candidates to FETCH so the bounded scan has something to choose
+     * from. Same rule the planner applies to the attempt window, so fetch and
+     * scan agree on one bound.
+     */
+    private candidateFetchLimit;
     /**
      * Cell identity for this handle() call. Set only for a single-work cell of a
      * scheduled occurrence; null for ad-hoc runs and N-works-per-run targets, which
@@ -26,7 +49,15 @@ export declare class IllustrationTargetHandler {
     private execution;
     constructor(client: IPixivClient, database: IDatabase, rankingService: RankingService, illustrationDownloader: IllustrationDownloader, pipeline: DownloadPipeline, topicPipelineFactory?: TopicPipelineFactory | undefined, deliveryService?: DeliveryService | undefined);
     handle(target: TargetConfig, execution?: TargetExecutionContext): Promise<TargetOutcome>;
-    /** Reduce the outcomes collected while processing one target to one cell result. */
+    /**
+     * Reduce the outcomes collected while processing one target to one cell
+     * result, including the bounded-scan bookkeeping.
+     *
+     * A `duplicate` is deliberately NOT a target verdict here: a duplicate is a
+     * CANDIDATE problem (skip it and try the next candidate), which is what the
+     * scan already did. Returning it as the target outcome is the bug that made a
+     * scheduled slot report success after submitting nothing.
+     */
     private summarize;
     private classifyError;
     private fetchIllustrations;
@@ -56,6 +87,11 @@ export declare class IllustrationTargetHandler {
     /**
      * Process ONE candidate work for this cell.
      *
+     * Returns what happened to THIS candidate, so the pipeline can advance to the
+     * next one when it was unusable. Throwing is reserved for JOB-level failures
+     * (dead database / dead token / dead delivery provider): a candidate-level
+     * failure is reported as a `skipped` attempt and never as a job verdict.
+     *
      * The cell is bound to `illust` BEFORE any side effect: it is the binding, not
      * the candidate list, that decides what a later recovery resumes. The binding
      * is only rolled back when the attempt produced no artifact at all, so in-run
@@ -67,6 +103,11 @@ export declare class IllustrationTargetHandler {
      * delivery mode the DeliveryService creates the durable intent atomically and
      * the result is 'delivery_pending' (NOT submitted — the OutboxWorker confirms
      * the ACK). Persistent/download-only runs are 'stored'.
+     *
+     * An ALREADY-DELIVERED work is returned as a candidate SKIP, never as a
+     * target outcome: the run must try the next candidate instead of ending the
+     * slot as a `duplicate`. The delivery idempotency ledger is what makes the
+     * second concurrent worker lose this race instead of double-submitting.
      */
     private recordArtifactOutcome;
     private executionContextFields;