pixivflow 2.19.3 → 2.19.5

package/dist/commands/scheduler-runtime.js

@@ -59,6 +59,7 @@ const schedules_1 = require("../scheduler/schedules");
 const SlotCoordinator_1 = require("../scheduler/SlotCoordinator");
 const DeliveryLedgerPort_1 = require("../delivery/DeliveryLedgerPort");
 const OutboxWorker_1 = require("../delivery/OutboxWorker");
+const settleDeliveryTerminal_1 = require("../delivery/settleDeliveryTerminal");
 const LegacyOutboxMigration_1 = require("../delivery/LegacyOutboxMigration");
 const NotificationPolicy_1 = require("../notification/NotificationPolicy");
 const node_crypto_1 = require("node:crypto");
@@ -241,22 +242,10 @@ async function createSchedulerRuntime(configPathArg) {
     const outboxWorker = new OutboxWorker_1.OutboxWorker(database, deliveryDispatcher, {
         retryBaseMs: config.delivery?.outboxRetryBaseMs,
         retryMaxMs: config.delivery?.outboxRetryMaxMs,
-        // A confirmed ACK promotes the delivery_pending cell to submitted.
+        // A confirmed ACK settles the owning Slot cell (submitted / duplicate /
+        // failed); see settleDeliveryTerminal for the invariant it enforces.
         onDeliveryTerminal: (deliveryId, ack) => {
-            const row = database.deliveries.getById(deliveryId);
-            if (!row || !row.slotId || !row.targetId)
-                return;
-            if (ack.kind === 'duplicate_existing') {
-                const coord = new SlotCoordinator_1.SlotCoordinator(database);
-                coord.applyOutcome(row.slotId, row.targetId, {
-                    kind: 'duplicate',
-                    workId: row.pixivId,
-                    reason: 'downstream attested historical duplicate',
-                });
-                return;
-            }
-            const coord = new SlotCoordinator_1.SlotCoordinator(database);
-            coord.markDelivered(row.slotId, row.targetId, row.pixivId, row.workType);
+            (0, settleDeliveryTerminal_1.settleDeliveryTerminal)(database, deliveryId, ack);
         },
     });
     const notificationPolicy = new NotificationPolicy_1.NotificationPolicy(database, config);

package/dist/config/defaults.d.ts

@@ -38,6 +38,7 @@ export declare const DEFAULT_CONFIG: {
         readonly maxRetries: 3;
         readonly retryDelay: 2000;
         readonly timeout: 60000;
+        readonly candidateScanLimit: 5;
     };
     readonly initialDelay: 0;
 };

package/dist/config/defaults.js

@@ -41,6 +41,10 @@ exports.DEFAULT_CONFIG = {
         maxRetries: 3,
         retryDelay: 2000,
         timeout: 60000,
+        // Candidates one execution may ATTEMPT while looking for an eligible work.
+        // Bounded so a ranking page full of already-delivered works cannot loop,
+        // but large enough that a single duplicate no longer burns the slot.
+        candidateScanLimit: 5,
     },
     initialDelay: 0,
 };

package/dist/config/environment.js

@@ -101,6 +101,25 @@ function applyEnvironmentOverrides(config) {
             overridden.scheduler.enabled = process.env.PIXIV_SCHEDULER_ENABLED.toLowerCase() === 'true';
         }
     }
+    // Override the bounded candidate-scan window. A non-numeric or non-positive
+    // value is ignored (the config-file value stays authoritative) rather than
+    // silently becoming NaN, which would disable the bound entirely.
+    if (process.env.PIXIV_CANDIDATE_SCAN_LIMIT !== undefined) {
+        const parsed = Number.parseInt(process.env.PIXIV_CANDIDATE_SCAN_LIMIT, 10);
+        if (Number.isFinite(parsed) && parsed > 0) {
+            if (!overridden.download) {
+                overridden.download = { candidateScanLimit: parsed };
+            }
+            else {
+                overridden.download.candidateScanLimit = parsed;
+            }
+        }
+        else {
+            logger_1.logger.warn('Ignoring PIXIV_CANDIDATE_SCAN_LIMIT: expected a positive integer', {
+                value: process.env.PIXIV_CANDIDATE_SCAN_LIMIT,
+            });
+        }
+    }
     // Override proxy from environment variables
     // Priority: all_proxy > https_proxy > http_proxy
     const proxyUrl = process.env.all_proxy || process.env.ALL_PROXY ||

package/dist/config/types.d.ts

@@ -81,6 +81,20 @@ export interface TargetConfig {
      * Maximum works to download per execution for this tag.
      */
     limit?: number;
+    /**
+     * Maximum CANDIDATE works one execution may attempt (not produce) while
+     * looking for an eligible work. A scheduled "one post per slot" target has
+     * `limit: 1`, so without this the run would hold a single candidate: if that
+     * one candidate is already delivered / deleted / private, the slot ends with
+     * nothing submitted. The scan walks up to N candidates in ranking order,
+     * skipping unusable ones, and is strictly bounded so a page full of
+     * duplicates cannot loop.
+     *
+     * Overrides `download.candidateScanLimit` / `PIXIV_CANDIDATE_SCAN_LIMIT`.
+     * Clamped to 1..100 and never below `limit`, so a multi-work target can still
+     * fill its own limit.
+     */
+    candidateScanLimit?: number;
     /**
      * Search target parameter for Pixiv API.
      */
@@ -655,6 +669,13 @@ export interface StandaloneConfig {
          * Default: 60000
          */
         timeout?: number;
+        /**
+         * How many CANDIDATE works one execution may attempt while looking for an
+         * eligible one. Per-target `candidateScanLimit` overrides this.
+         *
+         * Default: 5 (clamped to 1..100)
+         */
+        candidateScanLimit?: number;
     };
 }
 //# sourceMappingURL=types.d.ts.map

package/dist/config/validation.js

@@ -469,6 +469,21 @@ function validateConfig(config, location, databasePath) {
         if (config.download.maxRetries !== undefined && (config.download.maxRetries < 0 || config.download.maxRetries > 10)) {
             warnings.push('download.maxRetries: Should be between 0 and 10');
         }
+        if (config.download.candidateScanLimit !== undefined &&
+            (!Number.isInteger(config.download.candidateScanLimit) ||
+                config.download.candidateScanLimit < 1 ||
+                config.download.candidateScanLimit > 100)) {
+            warnings.push('download.candidateScanLimit: Should be an integer between 1 and 100');
+        }
+    }
+    // The bounded candidate scan is what stops a page full of duplicates from
+    // burning a whole scheduled slot, so a non-integer value is reported here.
+    for (const target of config.targets ?? []) {
+        if (target.candidateScanLimit === undefined)
+            continue;
+        if (!Number.isInteger(target.candidateScanLimit) || target.candidateScanLimit < 1 || target.candidateScanLimit > 100) {
+            warnings.push(`targets.${target.id ?? target.tag ?? '?'}.candidateScanLimit: Should be an integer between 1 and 100`);
+        }
     }
     // Validate log level
     if (config.logLevel && !['debug', 'info', 'warn', 'error'].includes(config.logLevel)) {

package/dist/delivery/DeliveryAck.d.ts

@@ -35,6 +35,20 @@ export type DeliveryAck = {
     matchedKey?: string;
     raw?: unknown;
 }
+/**
+ * The provider accepted the request and persisted a record, but that record
+ * is in a TERMINAL FAILURE state (`failed`/`rejected`/`invalid`/`expired`).
+ * Nothing will ever be published from this intent, and because the provider
+ * keys its record by OUR idempotency_key, another attempt only returns the
+ * same failed record. Terminal: never retried, never reported as delivered.
+ */
+ | {
+    kind: 'remote_failed';
+    remoteId?: string;
+    remoteStatus: string;
+    error: string;
+    raw?: unknown;
+}
 /** Transient failure: timeouts, 5xx, 429, connection errors. Retryable. */
  | {
     kind: 'retryable_failure';

package/dist/delivery/DeliveryAck.js

@@ -1,6 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.parseDeliveryAck = parseDeliveryAck;
+/** Downstream record states that will never become a published delivery. */
+const TERMINAL_REMOTE_STATUSES = new Set(['failed', 'rejected', 'invalid', 'expired']);
 const DEFAULT_HINTS = {
     dataPath: 'data',
     idField: 'review_id',
@@ -49,6 +51,22 @@ function parseDeliveryAck(status, body, hints = {}) {
     const remoteId = typeof idValue === 'number' ? String(idValue) : asString(idValue);
     const remoteStatus = asString(rec[h.statusField]);
     const reused = rec[h.reusedField] === true;
+    // A 2xx envelope can still describe a RECORD THAT FAILED downstream: the
+    // provider persists the record before doing the real work, so "accepted" and
+    // "the persisted record is broken" arrive on the same HTTP status. The record
+    // is what gets published (and what a retry will find again), so its terminal
+    // state outranks the transport-level success of this request — otherwise a
+    // remote failure is recorded here as an end-to-end success.
+    const terminalStatus = remoteStatus?.trim().toLowerCase();
+    if (terminalStatus && TERMINAL_REMOTE_STATUSES.has(terminalStatus)) {
+        return {
+            kind: 'remote_failed',
+            remoteId,
+            remoteStatus: terminalStatus,
+            error: `delivery endpoint reported terminal status ${terminalStatus}`,
+            raw: body,
+        };
+    }
     if (!reused) {
         return { kind: 'accepted', remoteId, remoteStatus, raw: body };
     }

package/dist/delivery/DeliveryService.d.ts

@@ -35,6 +35,15 @@ export declare class DeliveryService {
     isAlreadyDelivered(deliveryTarget: string, workType: string, pixivId: string): boolean;
     /** Batch form for pre-lock candidate filtering. */
     deliveredIds(deliveryTarget: string, workType: string, pixivIds: string[]): Set<string>;
+    /**
+     * Bulk CANDIDATE SELECTION dedupe: works already delivered to this target, or
+     * whose review submission is still PENDING an answer. A pending work is
+     * already submitted for review, so it must not be selected (and submitted)
+     * again; the scan moves on to the next candidate instead. Within-slot RESUME
+     * keeps using `isAlreadyDelivered` — a cell continuing its OWN pending work is
+     * resuming it, not duplicating it.
+     */
+    submittedIds(deliveryTarget: string, workType: string, pixivIds: string[]): Set<string>;
     /**
      * Atomically create the delivery intent + outbox row and advance the cell.
      * Missing local artifact files are treated as a recoverable error (the

package/dist/delivery/DeliveryService.js

@@ -33,6 +33,17 @@ class DeliveryService {
     deliveredIds(deliveryTarget, workType, pixivIds) {
         return this.database.deliveries.deliveredIds(deliveryTarget, workType, pixivIds);
     }
+    /**
+     * Bulk CANDIDATE SELECTION dedupe: works already delivered to this target, or
+     * whose review submission is still PENDING an answer. A pending work is
+     * already submitted for review, so it must not be selected (and submitted)
+     * again; the scan moves on to the next candidate instead. Within-slot RESUME
+     * keeps using `isAlreadyDelivered` — a cell continuing its OWN pending work is
+     * resuming it, not duplicating it.
+     */
+    submittedIds(deliveryTarget, workType, pixivIds) {
+        return this.database.deliveries.submittedIds(deliveryTarget, workType, pixivIds);
+    }
     /**
      * Atomically create the delivery intent + outbox row and advance the cell.
      * Missing local artifact files are treated as a recoverable error (the

package/dist/delivery/OutboxWorker.js

@@ -312,6 +312,19 @@ class OutboxWorker {
                 });
                 this.onDeliveryTerminal?.(row.deliveryId, ack, payload);
                 break;
+            case 'remote_failed':
+                // The provider persisted the record, but it will never publish and the
+                // idempotency key pins us to that same record forever: retrying cannot
+                // change the outcome. Terminal, so the failure reaches the Slot instead
+                // of being masked as a delivered cell. Content is kept for inspection.
+                this.database.deliveries.recordAck(row.deliveryId, {
+                    status: 'failed',
+                    remoteId: ack.remoteId,
+                    remoteStatus: ack.remoteStatus,
+                    error: ack.error,
+                });
+                this.onDeliveryTerminal?.(row.deliveryId, ack, payload);
+                break;
             case 'permanent_failure':
                 // Deterministic rejection: still retry a couple times to survive a
                 // misconfigured blip, the outbox max-attempts then dead-letters it.

package/dist/delivery/settleDeliveryTerminal.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Terminal-ACK settlement: the ONE place a confirmed downstream delivery ACK
+ * becomes a Slot cell state.
+ *
+ * The outbox worker already recorded the durable delivery ledger outcome before
+ * calling here; this maps that ack onto the Slot FSM. Keeping it as a named,
+ * dependency-light function (instead of an inline closure in the scheduler
+ * runtime) is what makes the central invariant directly testable:
+ *
+ *   REMOTE FAILURE MUST NOT BE REPORTED AS END-TO-END SUCCESS.
+ *
+ * A provider can answer HTTP 2xx while the record it persisted is terminally
+ * broken. The provider keys that record by OUR idempotency key, so a retry only
+ * returns the same broken record. Such an ack must settle the cell as `failed`,
+ * never promote it to `submitted` as though the content had been published.
+ */
+import type { Database } from '../storage/Database';
+import type { DeliveryAck } from './DeliveryAck';
+/**
+ * Apply a terminal delivery ack to the Slot cell that owns the delivery intent.
+ * Returns false when the delivery has no Slot cell (ad-hoc / batch runs), which
+ * is a normal no-op rather than an error.
+ */
+export declare function settleDeliveryTerminal(database: Database, deliveryId: string, ack: DeliveryAck): boolean;
+//# sourceMappingURL=settleDeliveryTerminal.d.ts.map

package/dist/delivery/settleDeliveryTerminal.js

@@ -0,0 +1,40 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.settleDeliveryTerminal = settleDeliveryTerminal;
+const SlotCoordinator_1 = require("../scheduler/SlotCoordinator");
+/**
+ * Apply a terminal delivery ack to the Slot cell that owns the delivery intent.
+ * Returns false when the delivery has no Slot cell (ad-hoc / batch runs), which
+ * is a normal no-op rather than an error.
+ */
+function settleDeliveryTerminal(database, deliveryId, ack) {
+    const row = database.deliveries.getById(deliveryId);
+    if (!row || !row.slotId || !row.targetId)
+        return false;
+    const coord = new SlotCoordinator_1.SlotCoordinator(database);
+    if (ack.kind === 'duplicate_existing') {
+        // Historical duplicate: a record from a DIFFERENT intent already exists, so
+        // THIS intent published nothing. Never a successful submission.
+        coord.applyOutcome(row.slotId, row.targetId, {
+            kind: 'duplicate',
+            workId: row.pixivId,
+            reason: 'downstream attested historical duplicate',
+        });
+        return true;
+    }
+    if (ack.kind === 'remote_failed') {
+        // The downstream record is terminally broken, so nothing was published:
+        // settle the cell as failed instead of promoting it to submitted. Not
+        // retryable — the same idempotency key can only return that record.
+        coord.applyOutcome(row.slotId, row.targetId, {
+            kind: 'failed',
+            retryable: false,
+            error: `downstream reported terminal status ${ack.remoteStatus} (review ${ack.remoteId ?? 'unknown'})`,
+        });
+        return true;
+    }
+    // accepted / idempotent_replay: a confirmed ACK is the sole path to submitted.
+    coord.markDelivered(row.slotId, row.targetId, row.pixivId, row.workType);
+    return true;
+}
+//# sourceMappingURL=settleDeliveryTerminal.js.map

package/dist/download/DownloadManager.js

@@ -128,6 +128,9 @@ class DownloadManager {
         this.novelDownloader = new NovelDownloader_1.NovelDownloader(client, database, fileService, database);
         this.planner = new DownloadPlanner_1.DownloadPlanner(database, {
             deliveredIds: (target, type, ids) => this.deliveryService.deliveredIds(target, type, ids),
+            // CANDIDATE SELECTION dedupe: also treats a work whose review submission is
+            // still PENDING as taken, so it is skipped instead of submitted again.
+            submittedIds: (target, type, ids) => this.deliveryService.submittedIds(target, type, ids),
             // Durable duplicate history handed in by the caller (the batch runner asks
             // the control plane for it). Independent of any local delivery target, so it
             // also works for a shadow run that delivers nowhere.
@@ -137,7 +140,10 @@ class DownloadManager {
                     return new Set();
                 return new Set(ids.filter((id) => known.has(id)));
             },
-        });
+            // Global bounded candidate-scan window (`download.candidateScanLimit`).
+            // Per-target `candidateScanLimit` overrides it. Without this the pipeline
+            // would only ever see as many candidates as the target's own `limit`.
+        }, config.download?.candidateScanLimit);
         this.executor = new DownloadExecutor_1.DownloadExecutor();
         const downloadConfig = config.download ?? {};
         const maxRetries = downloadConfig.maxRetries ?? 3;
@@ -166,6 +172,11 @@ class DownloadManager {
             : config.storage?.databasePath, config.download?.requestDelay ?? 500, this.abortController.signal);
         this.illustrationHandler = new IllustrationTargetHandler_1.IllustrationTargetHandler(client, database, this.rankingService, this.illustrationDownloader, this.pipeline, topicFactory, this.deliveryService);
         this.novelHandler = new NovelTargetHandler_1.NovelTargetHandler(client, database, this.rankingService, this.pipeline, this.novelDownloader, topicFactory, this.deliveryService);
+        // The FETCH stage needs the same bound the planner applies to the attempt
+        // window, or a `limit: 1` target would ask the ranking API for one work and
+        // leave the bounded scan nothing to scan.
+        this.illustrationHandler.setDefaultCandidateScanLimit(config.download?.candidateScanLimit);
+        this.novelHandler.setDefaultCandidateScanLimit(config.download?.candidateScanLimit);
     }
     setProgressCallback(callback) {
         this.progressReporter.setCallback(callback);

package/dist/download/handlers/IllustrationTargetHandler.d.ts

@@ -18,6 +18,29 @@ export declare class IllustrationTargetHandler {
     private readonly deliveryService?;
     /** Outcomes produced during this handle() call (deliveries + terminal non-matches). */
     private outcomes;
+    /**
+     * Candidate scan of the last pipeline.run() of this handle() call: how many
+     * candidates were attempted, which were skipped and why, and whether any
+     * job-level outage appeared. This is what turns an empty result into an
+     * explicit verdict instead of an ambiguous "completed".
+     */
+    private scan;
+    /**
+     * Global candidate-scan bound (`download.candidateScanLimit`), supplied by
+     * DownloadManager. This is what lets the FETCH stage ask for more than one
+     * candidate: a scheduled one-post-per-slot target has `limit: 1`, and asking
+     * the ranking API for exactly one work is the reason a single duplicate used
+     * to be unfixable downstream.
+     */
+    private defaultCandidateScanLimit?;
+    /** Publish the global candidate-scan bound for the fetch stage. */
+    setDefaultCandidateScanLimit(limit: number | undefined): void;
+    /**
+     * How many candidates to FETCH so the bounded scan has something to choose
+     * from. Same rule the planner applies to the attempt window, so fetch and
+     * scan agree on one bound.
+     */
+    private candidateFetchLimit;
     /**
      * Cell identity for this handle() call. Set only for a single-work cell of a
      * scheduled occurrence; null for ad-hoc runs and N-works-per-run targets, which
@@ -26,7 +49,15 @@ export declare class IllustrationTargetHandler {
     private execution;
     constructor(client: IPixivClient, database: IDatabase, rankingService: RankingService, illustrationDownloader: IllustrationDownloader, pipeline: DownloadPipeline, topicPipelineFactory?: TopicPipelineFactory | undefined, deliveryService?: DeliveryService | undefined);
     handle(target: TargetConfig, execution?: TargetExecutionContext): Promise<TargetOutcome>;
-    /** Reduce the outcomes collected while processing one target to one cell result. */
+    /**
+     * Reduce the outcomes collected while processing one target to one cell
+     * result, including the bounded-scan bookkeeping.
+     *
+     * A `duplicate` is deliberately NOT a target verdict here: a duplicate is a
+     * CANDIDATE problem (skip it and try the next candidate), which is what the
+     * scan already did. Returning it as the target outcome is the bug that made a
+     * scheduled slot report success after submitting nothing.
+     */
     private summarize;
     private classifyError;
     private fetchIllustrations;
@@ -56,6 +87,11 @@ export declare class IllustrationTargetHandler {
     /**
      * Process ONE candidate work for this cell.
      *
+     * Returns what happened to THIS candidate, so the pipeline can advance to the
+     * next one when it was unusable. Throwing is reserved for JOB-level failures
+     * (dead database / dead token / dead delivery provider): a candidate-level
+     * failure is reported as a `skipped` attempt and never as a job verdict.
+     *
      * The cell is bound to `illust` BEFORE any side effect: it is the binding, not
      * the candidate list, that decides what a later recovery resumes. The binding
      * is only rolled back when the attempt produced no artifact at all, so in-run
@@ -67,6 +103,11 @@ export declare class IllustrationTargetHandler {
      * delivery mode the DeliveryService creates the durable intent atomically and
      * the result is 'delivery_pending' (NOT submitted — the OutboxWorker confirms
      * the ACK). Persistent/download-only runs are 'stored'.
+     *
+     * An ALREADY-DELIVERED work is returned as a candidate SKIP, never as a
+     * target outcome: the run must try the next candidate instead of ending the
+     * slot as a `duplicate`. The delivery idempotency ledger is what makes the
+     * second concurrent worker lose this race instead of double-submitting.
      */
     private recordArtifactOutcome;
     private executionContextFields;