pixivflow 2.19.3 → 2.19.5

package/dist/download/handlers/NovelTargetHandler.js

@@ -5,8 +5,10 @@ const logger_1 = require("../../logger");
 const errors_1 = require("../../utils/errors");
 const pixiv_date_utils_1 = require("../../utils/pixiv-date-utils");
 const pixiv_utils_1 = require("../../utils/pixiv-utils");
+const TargetOutcome_1 = require("../../scheduler/TargetOutcome");
 const WorkIdentity_1 = require("../../scheduler/WorkIdentity");
 const target_label_1 = require("../../utils/target-label");
+const DownloadPlanner_1 = require("../plan/DownloadPlanner");
 class NovelTargetHandler {
     client;
     database;
@@ -16,6 +18,34 @@ class NovelTargetHandler {
     topicPipelineFactory;
     deliveryService;
     outcomes = [];
+    /**
+     * Candidate scan of the last pipeline.run() of this handle() call: how many
+     * candidates were attempted, which were skipped and why, and whether any
+     * job-level outage appeared. This is what turns an empty result into an
+     * explicit verdict instead of an ambiguous "completed".
+     */
+    scan = null;
+    /**
+     * Global candidate-scan bound (`download.candidateScanLimit`), supplied by
+     * DownloadManager. This is what lets the FETCH stage ask for more than one
+     * candidate: a scheduled one-post-per-slot target has `limit: 1`, and asking
+     * the ranking API for exactly one work is the reason a single duplicate used
+     * to be unfixable downstream.
+     */
+    defaultCandidateScanLimit;
+    /** Publish the global candidate-scan bound for the fetch stage. */
+    setDefaultCandidateScanLimit(limit) {
+        this.defaultCandidateScanLimit = limit;
+    }
+    /**
+     * How many candidates to FETCH so the bounded scan has something to choose
+     * from. Same rule the planner applies to the attempt window, so fetch and
+     * scan agree on one bound.
+     */
+    candidateFetchLimit(target) {
+        const targetLimit = target.limit && target.limit > 0 ? target.limit : 10;
+        return Math.max(targetLimit, (0, DownloadPlanner_1.resolveCandidateScanLimit)(target, this.defaultCandidateScanLimit));
+    }
     /**
      * Cell identity for this handle() call. Set only for a single-work cell of a
      * scheduled occurrence; null for ad-hoc runs and N-works-per-run targets, which
@@ -33,6 +63,7 @@ class NovelTargetHandler {
     }
     async handle(target, execution) {
         this.outcomes = [];
+        this.scan = null;
         this.execution = execution && (0, WorkIdentity_1.isSingleWorkCell)(target) ? execution : null;
         // A cell that already owns a work is in RECOVERY, not in a new selection.
         // Crash/shutdown recovery is not an intentional second run: running the
@@ -72,6 +103,7 @@ class NovelTargetHandler {
             }
             const novels = await this.fetchNovels(target, mode);
             const result = await this.pipeline.run(novels, target, 'novel', (novel, tag) => this.downloadAndDeliver(novel, tag, target));
+            this.scan = result.scan;
             await this.handleDownloadResult(result, target, mode, novels.length);
             return this.summarize();
         }
@@ -79,26 +111,84 @@ class NovelTargetHandler {
             return this.classifyError(error, displayTag, mode);
         }
     }
+    /**
+     * Reduce the outcomes collected while processing one target to one verdict,
+     * including the bounded-scan bookkeeping.
+     *
+     * A `duplicate` is deliberately NOT a target verdict for a scan: a duplicate
+     * is a CANDIDATE problem (skip it and try the next), which is what the scan
+     * already did. Returning it here is the bug that made a scheduled slot report
+     * success after submitting nothing. It remains a verdict only for the
+     * single-work RECOVERY path, whose cell identity is fixed.
+     */
     summarize() {
-        return (this.outcomes.find((o) => o.kind === 'submitted') ??
-            this.outcomes.find((o) => o.kind === 'stored') ??
-            this.outcomes.find((o) => o.kind === 'delivery_pending') ??
-            this.outcomes.find((o) => o.kind === 'duplicate') ??
-            this.outcomes.find((o) => o.kind === 'failed') ?? {
+        const scan = this.scan ?? undefined;
+        const submitted = this.outcomes.find((o) => o.kind === 'submitted');
+        if (submitted)
+            return scan ? { ...submitted, scan } : submitted;
+        const stored = this.outcomes.find((o) => o.kind === 'stored');
+        if (stored)
+            return scan ? { ...stored, scan } : stored;
+        const pending = this.outcomes.find((o) => o.kind === 'delivery_pending');
+        if (pending)
+            return scan ? { ...pending, scan } : pending;
+        const duplicate = this.outcomes.find((o) => o.kind === 'duplicate');
+        if (duplicate)
+            return scan ? { ...duplicate, scan } : duplicate;
+        const failed = this.outcomes.find((o) => o.kind === 'failed');
+        if (failed)
+            return scan ? { ...failed, scan } : failed;
+        if (scan && scan.outages.length > 0) {
+            return {
+                kind: 'failed',
+                retryable: true,
+                error: `job-level outage while scanning candidates: ${scan.outages.join(', ')}`,
+                scan,
+            };
+        }
+        if (scan && (0, TargetOutcome_1.hasTransientFailure)(scan)) {
+            return {
+                kind: 'failed',
+                retryable: true,
+                error: `candidate scan hit transient infrastructure failures ` +
+                    `(${scan.skipped.filter((s) => s.retryable).length} of ${scan.attempted} attempted)`,
+                scan,
+            };
+        }
+        if (scan && (scan.attempted > 0 || scan.skipped.length > 0)) {
+            return { kind: 'no_candidate', reason: (0, TargetOutcome_1.noEligibleCandidateText)(scan), scan };
+        }
+        return {
             kind: 'no_candidate',
             reason: 'no matching novel after filtering/dedupe',
-        });
+            ...(scan ? { scan } : {}),
+        };
     }
     classifyError(error, displayTag, mode) {
         const message = error instanceof Error ? error.message : String(error);
+        const scan = this.scan ?? undefined;
+        // A hard job-level outage is named as such and is never recorded as a
+        // no-candidate business outcome, whatever its message happens to look like.
+        const outage = (0, TargetOutcome_1.classifyJobLevelOutage)(error);
         this.database.logExecution(displayTag, 'novel', 'failed', message);
-        logger_1.logger.error(`Novel ${mode === 'ranking' ? 'ranking' : 'tag'} ${displayTag} failed`, { error: message });
+        logger_1.logger.error(`Novel ${mode === 'ranking' ? 'ranking' : 'tag'} ${displayTag} failed`, {
+            error: message,
+            ...(outage ? { jobLevelOutage: outage } : {}),
+        });
+        if (outage) {
+            return {
+                kind: 'failed',
+                retryable: true,
+                error: `job-level outage (${outage}): ${message}`,
+                ...(scan ? { scan } : {}),
+            };
+        }
         if (/no matching|all .*filtered|no_candidate|language filter/i.test(message)) {
-            return { kind: 'no_candidate', reason: message };
+            return { kind: 'no_candidate', reason: message, ...(scan ? { scan } : {}) };
         }
         const retryable = (0, errors_1.isRetryableNetworkError)(error) ||
             (error instanceof Error && /timeout|econn|enotfound|etimed|429|5\d\d/i.test(error.message));
-        return { kind: 'failed', retryable, error: message };
+        return { kind: 'failed', retryable, error: message, ...(scan ? { scan } : {}) };
     }
     async fetchNovels(target, mode) {
         if (mode === 'topic') {
@@ -137,6 +227,7 @@ class NovelTargetHandler {
             skipped: 0,
             alreadyDownloaded: 0,
             filteredOut: 0,
+            scan: { bound: 0, attempted: 0, skipped: [], outages: [] },
         };
         let totalFound = 0;
         for (let offset = 0; offset <= additionalDays && aggregate.downloaded < requested; offset++) {
@@ -157,11 +248,20 @@ class NovelTargetHandler {
             const novels = await this.fetchTopicNovels(attemptTarget);
             totalFound += novels.length;
             const result = await this.pipeline.run(novels, attemptTarget, 'novel', (novel, tag) => this.downloadAndDeliver(novel, tag, attemptTarget));
+            // The lookback loop is ONE bounded scan: accumulate every day's
+            // skips/outages so the verdict covers all candidates attempted.
+            aggregate.scan = (0, TargetOutcome_1.mergeScanSummaries)(aggregate.scan, result.scan);
             aggregate.downloaded += result.downloaded;
             aggregate.skipped += result.skipped;
             aggregate.alreadyDownloaded += result.alreadyDownloaded;
             aggregate.filteredOut += result.filteredOut;
+            if (aggregate.scan.outages.length > 0) {
+                // A dead token / dead database / dead network is not "no matching
+                // novel": stop looking back and let the job fail/retry.
+                break;
+            }
         }
+        this.scan = aggregate.scan;
         await this.handleDownloadResult(aggregate, target, 'topic', totalFound, checkedDays);
     }
     resolveTopicDay(target) {
@@ -191,12 +291,14 @@ class NovelTargetHandler {
                 endDate: rankingDate,
                 limit: Math.max(targetLimit * 20, 100),
             };
+            const fetchLimit = this.candidateFetchLimit(target);
             let novels = await this.client.searchNovels(searchTarget);
             logger_1.logger.info(`Found ${novels.length} novel(s) for ${rankingDate}`);
-            this.sortByPopularityAndLog(novels, targetLimit);
-            if (novels.length > targetLimit) {
-                novels = novels.slice(0, targetLimit);
-                logger_1.logger.info(`Selected top ${novels.length} novel(s) by popularity`);
+            this.sortByPopularityAndLog(novels, fetchLimit);
+            if (novels.length > fetchLimit) {
+                novels = novels.slice(0, fetchLimit);
+                logger_1.logger.info(`Selected top ${novels.length} novel(s) by popularity ` +
+                    `(to fill ${targetLimit}, candidate scan bound ${fetchLimit})`);
             }
             return novels;
         }
@@ -207,7 +309,9 @@ class NovelTargetHandler {
                 rankingDate = (0, pixiv_date_utils_1.getYesterdayDate)();
             }
             logger_1.logger.info(`Fetching ranking novels (mode: ${rankingMode}, date: ${rankingDate})`);
-            const novels = await this.rankingService.getRankingNovelsWithFallback(rankingMode, rankingDate, target.limit);
+            // Ask for the whole bounded scan window, not `limit`: the planner then
+            // narrows it to the candidates this run may attempt.
+            const novels = await this.rankingService.getRankingNovelsWithFallback(rankingMode, rankingDate, this.candidateFetchLimit(target));
             logger_1.logger.info(`Ranking API returned ${novels.length} novel(s)`);
             return novels;
         }
@@ -236,10 +340,19 @@ class NovelTargetHandler {
         const { downloaded, skipped, alreadyDownloaded, filteredOut } = result;
         const targetLimit = target.limit || 10;
         const tagForLog = (0, target_label_1.getTargetLabel)(target);
-        if (downloaded === 0 && targetLimit > 0) {
+        // The bounded scan produced its own explicit verdict when it attempted
+        // candidates or pre-filtered some, so the legacy "zero downloads" reporter
+        // is only the fallback for a scan that had nothing at all to look at.
+        const scanOwnsVerdict = result.scan.attempted > 0 || result.scan.skipped.length > 0;
+        if (downloaded === 0 && targetLimit > 0 && !scanOwnsVerdict) {
             await this.handleZeroDownloads(alreadyDownloaded, skipped, filteredOut, targetLimit, tagForLog, mode, target, totalFound, checkedDays, result.skipDetails);
             return;
         }
+        if (scanOwnsVerdict && downloaded > 0) {
+            logger_1.logger.info(`Candidate scan verdict for ${tagForLog}: submitted after skipping ` +
+                `${result.scan.skipped.length} candidate(s) of ${result.scan.attempted} attempted ` +
+                `(bound ${result.scan.bound})`);
+        }
         if (downloaded > 0 && downloaded < targetLimit * 0.5 && skipped > 0) {
             logger_1.logger.warn(`Only downloaded ${downloaded} out of ${targetLimit} requested novel(s). ${skipped} novel(s) were skipped due to 404 errors or other issues.`);
         }
@@ -375,6 +488,7 @@ class NovelTargetHandler {
                 return;
             }
             const result = await this.pipeline.run(novels, target, 'novel', (novel, tag) => this.downloadAndDeliver(novel, tag, target));
+            this.scan = result.scan;
             this.handleDownloadResult(result, target, 'user', novels.length);
         }
         catch (error) {
@@ -460,7 +574,15 @@ class NovelTargetHandler {
                 user: detail.user,
                 create_date: detail.create_date,
             };
-            await this.downloadAndDeliver(novel, `novel-${novelId}`, target);
+            const attempt = await this.downloadAndDeliver(novel, `novel-${novelId}`, target);
+            if (attempt.kind === 'skipped') {
+                // A cell that already owns a work cannot advance to another candidate —
+                // its identity is fixed. An already-delivered locked work is therefore a
+                // terminal business duplicate for THIS cell (it published nothing new).
+                this.outcomes.push(attempt.skip.code === 'duplicate'
+                    ? { kind: 'duplicate', workId: lockedWorkId, reason: attempt.skip.reason }
+                    : { kind: 'failed', retryable: false, error: `LOCKED_WORK_UNAVAILABLE: ${attempt.skip.reason}` });
+            }
         }
         catch (error) {
             const message = error instanceof Error ? error.message : String(error);
@@ -478,6 +600,11 @@ class NovelTargetHandler {
     /**
      * Process ONE candidate work for this cell.
      *
+     * Returns what happened to THIS candidate, so the pipeline can advance to the
+     * next one when it was unusable. Throwing is reserved for JOB-level failures
+     * (dead database / dead token / dead delivery provider): a candidate-level
+     * failure is reported as a `skipped` attempt and never as a job verdict.
+     *
      * The cell is bound to `novel` BEFORE any side effect: it is the binding, not
      * the candidate list, that decides what a later recovery resumes. The binding
      * is only rolled back when the attempt produced no artifact at all, so in-run
@@ -498,7 +625,14 @@ class NovelTargetHandler {
                     targetId: execution.targetId,
                     boundWorkId: binding.workId,
                 });
-                return;
+                return {
+                    kind: 'skipped',
+                    skip: {
+                        code: 'duplicate',
+                        workId,
+                        reason: `cell already owns work ${binding.workId} (concurrent selection)`,
+                    },
+                };
             }
         }
         let artifact;
@@ -509,33 +643,83 @@ class NovelTargetHandler {
             // Nothing was persisted, so the cell may still pick another candidate.
             if (execution && !recovering)
                 execution.release(workId);
-            throw error;
+            const failure = (0, TargetOutcome_1.classifyCandidateFailure)(error, workId);
+            if (failure.scope === 'job' || !(0, TargetOutcome_1.skipCandidateWithoutRetry)(failure.skip)) {
+                // Job-level outage, or a transient candidate failure: re-thrown so the
+                // queue's existing retry/backoff owns it and the scheduler records a JOB
+                // failure — never an exhausted candidate list.
+                throw error;
+            }
+            this.logError(error, `Candidate novel ${workId} skipped (${failure.skip.code})`);
+            return { kind: 'skipped', skip: failure.skip };
         }
         if (!artifact) {
             if (execution && !recovering)
                 execution.release(workId);
-            return;
+            return {
+                kind: 'skipped',
+                skip: { code: 'filtered', workId, reason: 'downloader declined this candidate (no artifact)' },
+            };
         }
-        // Committed: the artifact is durable and this work now defines the cell. Even
-        // if the delivery below throws, recovery must resume THIS work — never release.
+        // The artifact is durable, but this cell has NOT committed to it until the
+        // delivery below actually claims it. A candidate that turns out to be a
+        // duplicate must give the binding BACK, or `bind()` would refuse the next
+        // candidate and the scan could never advance — the whole point of this
+        // change. `releaseCellWork` itself refuses once the cell moved on
+        // (delivery_pending/submitted), so a committed identity stays stable.
+        const attempt = this.recordArtifactOutcome(artifact, target);
+        if (attempt.kind === 'skipped' && execution && !recovering) {
+            execution.release(workId);
+        }
+        return attempt;
+    }
+    /**
+     * Turn a downloaded novel artifact into the target's business outcome.
+     *
+     * An ALREADY-DELIVERED work is returned as a candidate SKIP, never as a
+     * target outcome: the run must try the next candidate instead of ending the
+     * slot as a `duplicate`. The delivery idempotency ledger is what makes the
+     * second concurrent worker lose this race instead of double-submitting.
+     */
+    recordArtifactOutcome(artifact, target) {
         const isDelivery = target.storageMode === 'cache' && target.delivery?.target?.trim();
         if (!isDelivery || !this.deliveryService) {
             this.outcomes.push({ kind: 'stored', workId: artifact.pixivId, workType: artifact.type });
-            return;
+            return { kind: 'selected', workId: artifact.pixivId, workType: artifact.type };
         }
         const ec = target.delivery;
         const slotId = ec?.executionContext?.slotId;
         if (this.deliveryService.isAlreadyDelivered(target.delivery.target, artifact.type, artifact.pixivId)) {
-            this.outcomes.push({ kind: 'duplicate', workId: artifact.pixivId, reason: 'already delivered to target (ledger)' });
-            return;
+            return {
+                kind: 'skipped',
+                skip: {
+                    code: 'duplicate',
+                    workId: artifact.pixivId,
+                    reason: 'already delivered to target (delivery ledger)',
+                },
+            };
         }
         const res = this.deliveryService.enqueue(artifact, target, {
             slotId,
             fields: target.delivery?.fields,
         });
-        this.outcomes.push(res.duplicate
-            ? { kind: 'duplicate', workId: artifact.pixivId, reason: 'already delivered (ledger)' }
-            : { kind: 'delivery_pending', workId: artifact.pixivId, workType: artifact.type, deliveryId: res.deliveryId });
+        if (res.duplicate) {
+            return {
+                kind: 'skipped',
+                skip: {
+                    code: 'duplicate',
+                    workId: artifact.pixivId,
+                    reason: 'already delivered (idempotency ledger)',
+                },
+            };
+        }
+        this.outcomes.push({
+            kind: 'delivery_pending',
+            workId: artifact.pixivId,
+            workType: artifact.type,
+            deliveryId: res.deliveryId,
+        });
+        return { kind: 'selected', workId: artifact.pixivId, workType: artifact.type };
     }
 }
 exports.NovelTargetHandler = NovelTargetHandler;

package/dist/download/pipeline/DownloadPipeline.d.ts

@@ -4,10 +4,18 @@ import { DownloadPlanner } from '../plan/DownloadPlanner';
 import { DownloadExecutor } from '../exec/DownloadExecutor';
 import { ProgressReporter } from '../report/ProgressReporter';
 import { ErrorRecoveryStrategy } from '../recovery/ErrorRecovery';
+import { CandidateAttempt, CandidateScanSummary } from '../../scheduler/TargetOutcome';
 type DownloadItem = PixivIllust | PixivNovel;
 type ItemType = 'illustration' | 'novel';
 export interface DownloadPipelineResult {
+    /**
+     * Candidates that PRODUCED a business result for this target. A candidate
+     * that was skipped (duplicate, deleted, private, ...) does not count: that is
+     * precisely the accounting error that let a duplicate end a scheduled slot as
+     * a completed run.
+     */
     downloaded: number;
+    /** Candidates the executor skipped after an error (legacy error counter). */
     skipped: number;
     alreadyDownloaded: number;
     filteredOut: number;
@@ -16,6 +24,11 @@ export interface DownloadPipelineResult {
         id: string;
         error: string;
     }[];
+    /**
+     * Bounded candidate-scan bookkeeping. Always present, so the caller can state
+     * an explicit terminal outcome instead of "completed".
+     */
+    scan: CandidateScanSummary;
 }
 export interface DownloadPipelineDependencies {
     config: StandaloneConfig;
@@ -29,6 +42,14 @@ export interface DownloadPipelineDependencies {
      */
     isCancelled?: () => boolean;
 }
+/**
+ * What a candidate hook must report back: did this candidate produce the
+ * target's business result, or was it unusable?
+ *
+ * `void` is accepted and treated as `selected`, so a hook with no
+ * candidate-level information keeps the historical meaning of "it resolved".
+ */
+export type CandidateDownloadFn<T extends DownloadItem> = (item: T, tag: string) => Promise<CandidateAttempt | void>;
 export declare class DownloadPipeline {
     private readonly config;
     private readonly planner;
@@ -37,9 +58,7 @@ export declare class DownloadPipeline {
     private readonly recovery;
     private readonly isRunCancelled;
     constructor(deps: DownloadPipelineDependencies);
-    run<T extends DownloadItem>(items: T[], target: TargetConfig, itemType: ItemType, downloadFn: (item: T, tag: string) => Promise<void>): Promise<DownloadPipelineResult>;
-    private executeRandomMode;
-    private executeSequentialMode;
+    run<T extends DownloadItem>(items: T[], target: TargetConfig, itemType: ItemType, downloadFn: CandidateDownloadFn<T>): Promise<DownloadPipelineResult>;
     private updateProgress;
     private logRecoveryDecision;
 }