pixivflow 3.2.0 → 3.4.0

package/dist/delivery/OutboxWorker.js

@@ -6,6 +6,7 @@ const node_crypto_1 = require("node:crypto");
 const promises_1 = require("node:fs/promises");
 const errorClass_1 = require("./errorClass");
 const redact_1 = require("../utils/redact");
+const ProtocolErrors_1 = require("../scheduler/ProtocolErrors");
 const logger_1 = require("../logger");
 /** Exponential backoff with jitter, capped. */
 function backoffDelayMs(attempt, base, max) {
@@ -81,6 +82,10 @@ class OutboxWorker {
                 break;
             let deferred = false;
             for (const row of rows) {
+                if (this.stopCancelledWork(row)) {
+                    dead++;
+                    continue;
+                }
                 const readiness = await this.checkReadiness(row);
                 if (!readiness.ready) {
                     this.database.outbox.release(row.id);
@@ -114,6 +119,8 @@ class OutboxWorker {
                     this.database.outbox.release(row.id);
                     continue;
                 }
+                if (this.stopCancelledWork(row))
+                    continue;
                 const readiness = await this.checkReadiness(row);
                 if (!readiness.ready) {
                     this.database.outbox.release(row.id);
@@ -138,6 +145,36 @@ class OutboxWorker {
         }
         return { ready: await this.dispatcher.isReady(row.deliveryTarget) };
     }
+    /**
+     * Honour a consumer cancel.
+     *
+     * A job cancelled while a delivery attempt was in flight (or waiting to
+     * retry) must not be delivered afterwards: the ledger's cell already carries
+     * the terminal `cancelled_by_consumer` verdict, so the intent behind this row
+     * is void. The row is dead-lettered rather than retried — no amount of
+     * retrying can make a cancelled delivery correct — and the skip is logged so
+     * the operator can see what did NOT happen.
+     */
+    stopCancelledWork(row) {
+        if (!row.deliveryId)
+            return false;
+        const delivery = this.database.deliveries.getById(row.deliveryId);
+        if (!delivery?.slotId || !delivery.targetId)
+            return false;
+        const cell = this.database.slots.getCell(delivery.slotId, delivery.targetId);
+        if (!cell || cell.status !== 'failed' || cell.terminalReasonCode !== ProtocolErrors_1.CANCELLED_BY_CONSUMER) {
+            return false;
+        }
+        const reason = 'work cancelled by consumer';
+        this.database.outbox.markDead(row.id, reason, Date.now());
+        logger_1.logger.warn('Delivery skipped: work cancelled by consumer', {
+            outboxId: row.id,
+            deliveryId: delivery.id,
+            slot: delivery.slotId,
+            target: delivery.targetId,
+        });
+        return true;
+    }
     /** Record at most one deferral per outbox row per 60s (cold-start poll guard). */
     recordDeferred(row, probe) {
         const now = Date.now();
@@ -223,6 +260,16 @@ class OutboxWorker {
                         : undefined,
                 });
             }
+            else if (row.kind === 'event_callback') {
+                // A `$defs/Event` posted to the Task's declared callback_url. It reuses
+                // the whole outbox contract: the idempotency key is the event_id, so a
+                // duplicate intent dedupes here and a retry re-sends the same event.
+                const payload = JSON.parse(row.payloadJson);
+                await this.dispatcher.deliverEventCallback(row.deliveryTarget, {
+                    payload,
+                    idempotencyKey: row.idempotencyKey ?? row.id,
+                });
+            }
             else {
                 const payload = JSON.parse(row.payloadJson);
                 const result = await this.dispatcher.deliver(row.deliveryTarget, {
@@ -258,7 +305,10 @@ class OutboxWorker {
         }
         catch (error) {
             const message = (0, redact_1.redactError)(error).slice(0, 1000);
-            const { errorClass, retryable } = (0, errorClass_1.classifyError)(error);
+            // A provider that answers with an HTTP status is classified by that status
+            // (429/5xx retry, other 4xx dead-letter); transport errors classify from
+            // their message as before.
+            const { errorClass, retryable } = (0, errorClass_1.classifyError)(error, error?.status);
             // Deterministic failures cannot improve under the same idempotent intent.
             // Retrying them only leaves the owning Slot running until the budget expires.
             if (!retryable) {

package/dist/package.json

@@ -1,6 +1,6 @@
 {
   "type": "commonjs",
   "name": "pixivflow",
-  "version": "3.2.0",
+  "version": "3.4.0",
   "private": true
 }

package/dist/scheduler/CandidateSearchParams.d.ts

@@ -0,0 +1,92 @@
+/**
+ * Occurrence-scoped retrieval view for `candidate_search` (§6 `$defs/CandidateSearchParams`).
+ *
+ * A generic job may narrow or redirect HOW it looks for a work — tags, expansion,
+ * scan depth, exclusions — but never WHAT it delivers: `applyCandidateSearchParams`
+ * is a pure function over a target snapshot that touches retrieval levers only.
+ * Delivery wiring, the target id and the plan identity are structurally out of
+ * reach here, which is why this mapping lives in one small pure module instead of
+ * being spread over the run path.
+ *
+ * The requested view is stored on the manual Slot (the same occurrence-scoped
+ * precedent as `recovery_mode`) so a worker that resumes a crashed job re-applies
+ * exactly the retrieval the requester asked for. Nothing here writes global
+ * config: a manual job can never change future scheduled runs.
+ */
+import { TargetConfig } from '../config';
+/** `$defs/CandidateSearchParams.constraints.exclude` item. */
+export interface CandidateSearchExclude {
+    kind: 'work' | 'candidate' | 'tag';
+    id: string;
+}
+/** `$defs/CandidateSearchParams`. Only the retrieval/constraint part is modelled. */
+export interface CandidateSearchParams {
+    source?: {
+        platform?: string;
+        account?: string;
+    };
+    /**
+     * Scope selector: which of the producer's CONFIGURED targets this job runs
+     * for. It selects an existing target — it never overrides that target's
+     * delivery wiring or plan identity (see `ManualJobAdmission.resolveTarget`).
+     * It is deliberately NOT part of the stored retrieval view: the admission
+     * lifts it into `CandidateSearchJobRequest.targetSelector`, so a job that only
+     * names a target runs that target exactly as configured. This is what makes
+     * the generic `POST /jobs` face equivalent to the legacy refetch entry point,
+     * whose target used to live in the request path.
+     */
+    target_id?: string;
+    /**
+     * Retrieval override. Optional: omitting it (with or without `target_id`)
+     * means "run the resolved target's own configured search", which is what every
+     * pre-protocol refetch did.
+     */
+    query?: {
+        tags: string[];
+        expand?: boolean;
+    };
+    constraints?: {
+        exclude?: CandidateSearchExclude[];
+        limit?: number;
+        scan_limit?: number;
+        work_types?: string[];
+    };
+}
+/** Scan limits are clamped to the same 1..100 window the config validation uses. */
+export declare const CANDIDATE_SEARCH_SCAN_LIMIT_MAX = 100;
+/**
+ * Project one target through a requested retrieval view (pure; never mutates the
+ * input or the global config).
+ *
+ * - `query.tags` becomes the tag expression (`tag` is the repo's space-joined
+ *   multi-tag field).
+ * - `query.expand` maps to `tagRelation: 'or'`: for multiple tags that is exactly
+ *   "recall expansion" (union instead of intersection), and for a single tag it
+ *   is a no-op rather than an invented behaviour.
+ * - `constraints.limit` / `constraints.scan_limit` feed the existing per-target
+ *   knobs, clamped to the validated window and never below `limit`.
+ *
+ * `constraints.exclude` and `constraints.work_types` are NOT applied here: an
+ * exclusion is a run-level duplicate filter (`excludedWorkIdsFromParams`) and a
+ * work-type restriction is checked against the resolved target by the admission,
+ * where the target is known.
+ */
+export declare function applyCandidateSearchParams(target: TargetConfig, params: CandidateSearchParams): TargetConfig;
+/**
+ * Map `constraints.exclude` onto the existing per-run duplicate history.
+ *
+ * A `kind: 'work'` exclusion is a work id the runner must not select. The ledger
+ * keys history by work type, and a protocol caller does not know the producer's
+ * work type, so the id is excluded for both types — narrowing, never widening.
+ */
+export declare function excludedWorkIdsFromParams(params: CandidateSearchParams): {
+    illustration: string[];
+    novel: string[];
+} | null;
+/**
+ * Read the retrieval view back off a Slot row. Tolerant by design: a row written
+ * by an older build (or corrupted JSON) yields `null`, which means "run the plan
+ * exactly as configured" — the behaviour of every pre-existing slot.
+ */
+export declare function parseCandidateSearchParamsJson(json: string | null | undefined): CandidateSearchParams | null;
+//# sourceMappingURL=CandidateSearchParams.d.ts.map

package/dist/scheduler/CandidateSearchParams.js

@@ -0,0 +1,103 @@
+"use strict";
+/**
+ * Occurrence-scoped retrieval view for `candidate_search` (§6 `$defs/CandidateSearchParams`).
+ *
+ * A generic job may narrow or redirect HOW it looks for a work — tags, expansion,
+ * scan depth, exclusions — but never WHAT it delivers: `applyCandidateSearchParams`
+ * is a pure function over a target snapshot that touches retrieval levers only.
+ * Delivery wiring, the target id and the plan identity are structurally out of
+ * reach here, which is why this mapping lives in one small pure module instead of
+ * being spread over the run path.
+ *
+ * The requested view is stored on the manual Slot (the same occurrence-scoped
+ * precedent as `recovery_mode`) so a worker that resumes a crashed job re-applies
+ * exactly the retrieval the requester asked for. Nothing here writes global
+ * config: a manual job can never change future scheduled runs.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CANDIDATE_SEARCH_SCAN_LIMIT_MAX = void 0;
+exports.applyCandidateSearchParams = applyCandidateSearchParams;
+exports.excludedWorkIdsFromParams = excludedWorkIdsFromParams;
+exports.parseCandidateSearchParamsJson = parseCandidateSearchParamsJson;
+/** Scan limits are clamped to the same 1..100 window the config validation uses. */
+exports.CANDIDATE_SEARCH_SCAN_LIMIT_MAX = 100;
+function clamp(value, min, max) {
+    return Math.max(min, Math.min(max, value));
+}
+/**
+ * Project one target through a requested retrieval view (pure; never mutates the
+ * input or the global config).
+ *
+ * - `query.tags` becomes the tag expression (`tag` is the repo's space-joined
+ *   multi-tag field).
+ * - `query.expand` maps to `tagRelation: 'or'`: for multiple tags that is exactly
+ *   "recall expansion" (union instead of intersection), and for a single tag it
+ *   is a no-op rather than an invented behaviour.
+ * - `constraints.limit` / `constraints.scan_limit` feed the existing per-target
+ *   knobs, clamped to the validated window and never below `limit`.
+ *
+ * `constraints.exclude` and `constraints.work_types` are NOT applied here: an
+ * exclusion is a run-level duplicate filter (`excludedWorkIdsFromParams`) and a
+ * work-type restriction is checked against the resolved target by the admission,
+ * where the target is known.
+ */
+function applyCandidateSearchParams(target, params) {
+    const tags = params.query?.tags;
+    const limit = params.constraints?.limit !== undefined ? clamp(params.constraints.limit, 1, exports.CANDIDATE_SEARCH_SCAN_LIMIT_MAX) : target.limit;
+    const scanLimit = params.constraints?.scan_limit !== undefined
+        ? clamp(params.constraints.scan_limit, limit ?? 1, exports.CANDIDATE_SEARCH_SCAN_LIMIT_MAX)
+        : target.candidateScanLimit;
+    const constrained = {
+        ...target,
+        ...(limit !== undefined ? { limit } : {}),
+        ...(scanLimit !== undefined ? { candidateScanLimit: scanLimit } : {}),
+    };
+    // `query` is optional: a job may carry only a scope selector and/or
+    // constraints, in which case the retrieval side stays exactly as the target is
+    // configured. (`parseCandidateSearchParamsJson` already yields `null` for such
+    // a stored view, so this is the belt to that braces.)
+    if (!tags || tags.length === 0)
+        return constrained;
+    return {
+        ...constrained,
+        tag: tags.join(' '),
+        ...(tags.length > 1 && params.query?.expand === true ? { tagRelation: 'or' } : {}),
+    };
+}
+/**
+ * Map `constraints.exclude` onto the existing per-run duplicate history.
+ *
+ * A `kind: 'work'` exclusion is a work id the runner must not select. The ledger
+ * keys history by work type, and a protocol caller does not know the producer's
+ * work type, so the id is excluded for both types — narrowing, never widening.
+ */
+function excludedWorkIdsFromParams(params) {
+    const ids = (params.constraints?.exclude ?? [])
+        .filter((entry) => entry.kind === 'work')
+        .map((entry) => entry.id)
+        .filter((id) => id.length > 0);
+    if (ids.length === 0)
+        return null;
+    return { illustration: [...ids], novel: [...ids] };
+}
+/**
+ * Read the retrieval view back off a Slot row. Tolerant by design: a row written
+ * by an older build (or corrupted JSON) yields `null`, which means "run the plan
+ * exactly as configured" — the behaviour of every pre-existing slot.
+ */
+function parseCandidateSearchParamsJson(json) {
+    if (!json)
+        return null;
+    try {
+        const parsed = JSON.parse(json);
+        if (!parsed || typeof parsed !== 'object')
+            return null;
+        if (!Array.isArray(parsed.query?.tags) || parsed.query.tags.length === 0)
+            return null;
+        return parsed;
+    }
+    catch {
+        return null;
+    }
+}
+//# sourceMappingURL=CandidateSearchParams.js.map

package/dist/scheduler/JobCancellation.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Consumer-initiated cancellation of one job, in ONE transaction.
+ *
+ * The protocol has no separate "cancelled" ledger state on purpose (§11.1: no
+ * new cell-FSM state may be invented). A cancel is therefore expressed with the
+ * existing terminal cell state: the cell becomes `failed` carrying the
+ * dedicated terminal reason `cancelled_by_consumer`, which the taxonomy treats
+ * as an operator/consumer stop rather than a Pixiv failure, and which the job
+ * facade surfaces as protocol `status: "cancelled"`.
+ *
+ * Stopping delivery is part of the same intent: every actionable outbox row for
+ * the cancelled cells is cancelled inside the transaction, and the delivery
+ * worker refuses to (re)attempt a delivery whose cell carries the cancel
+ * reason — a cancel that is undone by a retry would be worse than no cancel.
+ */
+import type { Database } from '../storage/Database';
+/** The user-facing business message for a consumer cancel. */
+export declare const CANCELLED_BY_CONSUMER_MESSAGE = "\u4EFB\u52A1\u5DF2\u88AB\u53D6\u6D88";
+export interface JobCancellationResult {
+    slotId: string;
+    /** True when THIS call performed the terminal transition. */
+    cancelled: boolean;
+    /** True when the work was already terminal before this call (idempotent replay). */
+    alreadyTerminal: boolean;
+    /** Outbox intents this call stopped from ever being attempted again. */
+    stoppedDeliveries: number;
+}
+/**
+ * Cancel one job (a slot) by id. Idempotent: a second call observes a terminal
+ * slot and reports `alreadyTerminal` without touching the ledger.
+ *
+ * A slot that already delivered something is never downgraded to `failed`; it
+ * keeps its delivered cell and rolls up as `partial`.
+ */
+export declare function cancelConsumerJob(database: Database, slotId: string, now?: number): JobCancellationResult;
+//# sourceMappingURL=JobCancellation.d.ts.map

package/dist/scheduler/JobCancellation.js

@@ -0,0 +1,73 @@
+"use strict";
+/**
+ * Consumer-initiated cancellation of one job, in ONE transaction.
+ *
+ * The protocol has no separate "cancelled" ledger state on purpose (§11.1: no
+ * new cell-FSM state may be invented). A cancel is therefore expressed with the
+ * existing terminal cell state: the cell becomes `failed` carrying the
+ * dedicated terminal reason `cancelled_by_consumer`, which the taxonomy treats
+ * as an operator/consumer stop rather than a Pixiv failure, and which the job
+ * facade surfaces as protocol `status: "cancelled"`.
+ *
+ * Stopping delivery is part of the same intent: every actionable outbox row for
+ * the cancelled cells is cancelled inside the transaction, and the delivery
+ * worker refuses to (re)attempt a delivery whose cell carries the cancel
+ * reason — a cancel that is undone by a retry would be worse than no cancel.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CANCELLED_BY_CONSUMER_MESSAGE = void 0;
+exports.cancelConsumerJob = cancelConsumerJob;
+const ProtocolErrors_1 = require("./ProtocolErrors");
+const SlotStateMachine_1 = require("./SlotStateMachine");
+/** The user-facing business message for a consumer cancel. */
+exports.CANCELLED_BY_CONSUMER_MESSAGE = '任务已被取消';
+/**
+ * Cancel one job (a slot) by id. Idempotent: a second call observes a terminal
+ * slot and reports `alreadyTerminal` without touching the ledger.
+ *
+ * A slot that already delivered something is never downgraded to `failed`; it
+ * keeps its delivered cell and rolls up as `partial`.
+ */
+function cancelConsumerJob(database, slotId, now = Date.now()) {
+    return database.transaction(() => {
+        const slot = database.slots.getSlot(slotId);
+        if (!slot) {
+            throw new ProtocolErrors_1.ProtocolRequestError('invalid_params', 404, {
+                message: 'unknown job',
+                detail: { reason: 'unknown_job', job_id: slotId },
+            });
+        }
+        const cells = slot.targetIds
+            .map((targetId) => database.slots.getCell(slotId, targetId))
+            .filter((cell) => cell !== null);
+        const open = cells.filter((cell) => !SlotStateMachine_1.TERMINAL_CELL_STATES.has(cell.status));
+        if (open.length === 0) {
+            return {
+                slotId,
+                cancelled: false,
+                alreadyTerminal: true,
+                stoppedDeliveries: 0,
+            };
+        }
+        let stoppedDeliveries = 0;
+        for (const cell of open) {
+            for (const delivery of database.deliveries.listForSlotCell(slotId, cell.targetId)) {
+                if (!database.outbox.hasActionableDelivery(delivery.id))
+                    continue;
+                const row = database.outbox.listForDeliveryIds([delivery.id]).get(delivery.id);
+                if (row && database.outbox.cancel(row.id, now))
+                    stoppedDeliveries++;
+            }
+            database.slots.transitionCell(slotId, cell.targetId, 'failed', exports.CANCELLED_BY_CONSUMER_MESSAGE);
+            database.slots.setCellTerminalReason(slotId, cell.targetId, ProtocolErrors_1.CANCELLED_BY_CONSUMER, exports.CANCELLED_BY_CONSUMER_MESSAGE);
+        }
+        // Never report a slot with a confirmed delivery as failed.
+        const delivered = cells.some((cell) => cell.status === 'submitted');
+        database.slots.markSlotStatus(slotId, delivered ? 'partial' : 'failed', exports.CANCELLED_BY_CONSUMER_MESSAGE);
+        const lease = database.slots.getSlotLease(slotId);
+        if (lease.owner)
+            database.slots.releaseSlotLease(slotId, lease.owner);
+        return { slotId, cancelled: true, alreadyTerminal: false, stoppedDeliveries };
+    });
+}
+//# sourceMappingURL=JobCancellation.js.map

package/dist/scheduler/JobEventStream.d.ts

@@ -0,0 +1,94 @@
+/**
+ * The durable job event stream (Workflow Protocol v1 §events).
+ *
+ * The stream is a PROJECTION of `delivery_events` — the log the delivery
+ * subsystem already writes — plus ONE durable ack cursor
+ * (`job_event_cursors`). There is no second event queue and no second table
+ * family: `slot_id` is the job id, `ts` is the protocol `at`, and `event` is the
+ * internal kind translated by the single mapping in `JobFacade`.
+ *
+ * Three properties this module owns, because they are protocol obligations:
+ *
+ *  1. **A terminal job always has a terminal event.** `SlotCoordinator`'s
+ *     `execution.summary` hook only fires for success/partial/failed, and
+ *     cancelled/expired slots would never produce one. Instead of widening the
+ *     slot/business-status logic, the lifecycle events are *reconciled from the
+ *     durable projection*: whatever the ledger already implies must exist, is
+ *     materialised exactly once (mirroring `reconcileScheduleSummaries`). The
+ *     page is therefore never empty for a job that exists.
+ *
+ *  2. **Ack is monotonic, idempotent and side-effect free.** It writes one row
+ *     in `job_event_cursors` and nothing else. An unknown or older cursor is a
+ *     no-op that reports the durable state — never an error.
+ *
+ *  3. **A declared `callback_url` is never silently dropped.** The event row and
+ *     its callback intent are committed in ONE transaction, and a sweep over
+ *     durable rows repairs any job whose terminal event is still missing (for
+ *     example because the process died mid-flight). Retry/dead-lettering stay in
+ *     the existing outbox worker.
+ */
+import { ProtocolAckResult, ProtocolEventPage } from './JobFacade';
+import { JobViewDeps } from './JobView';
+/** One page of events. Bounded so a long-running job cannot stream unbounded JSON. */
+export declare const EVENTS_PAGE_LIMIT = 200;
+/** Bounded callback budget: a flaky third-party URL must converge, not retry forever. */
+export declare const EVENT_CALLBACK_MAX_ATTEMPTS = 8;
+export interface JobEventQuery {
+    after: string | null;
+    unackedOnly: boolean;
+}
+export declare class JobEventStream {
+    private readonly deps;
+    constructor(deps: JobViewDeps);
+    private get db();
+    /**
+     * `$defs/EventPage`. Throws 404 for an unknown job, exactly like
+     * `GET /jobs/:jobId`.
+     */
+    page(jobId: string, query: JobEventQuery): ProtocolEventPage;
+    /**
+     * `$defs/AckResult`. `ack_through` is monotonic and idempotent: replaying it,
+     * or sending an unknown/older cursor, is a no-op that reports what is durably
+     * stored. Acking never touches the slot ledger.
+     */
+    ack(jobId: string, body: unknown): ProtocolAckResult;
+    /**
+     * Materialise every lifecycle event the durable ledger already implies, at
+     * most once each, and enqueue the matching callback in the same transaction.
+     * `callbackUrl` is only read when THIS call is the one that first records the
+     * job's request: a replay never rewrites the declared callback.
+     */
+    reconcile(jobId: string, callbackUrl?: string | null): void;
+    /**
+     * Repair sweep for jobs that declared a callback and have no durable terminal
+     * event yet, so a process that died before the terminal reconcile still
+     * delivers its terminal callback without waiting for a poll. Only jobs that
+     * already own a durable `job.requested` row are in scope: that is exactly the
+     * set whose callback endpoint is known, and re-running it can never queue a
+     * second copy (the event_id is the outbox idempotency key). A job that never
+     * recorded its request event is repaired on the consumer's next read instead.
+     * Returns how many jobs it looked at, so a caller can assert it drains.
+     */
+    reconcileOutstanding(limit?: number): number;
+    private ensure;
+    /** The `callback_url` a job declared, read from its durable request event. */
+    private declaredCallbackUrl;
+    private enqueueCallback;
+    private eventBodyFor;
+    private requireJob;
+    /**
+     * `?after=` is a "give me everything after this" hint. An unrecognised or
+     * foreign cursor cannot filter, so it is treated as absent and the consumer
+     * sees the whole stream again — `event_id` is the dedupe key, and at-least-once
+     * is the contract.
+     */
+    private resolveAfter;
+    /**
+     * Map an opaque `event_id` back to a durable position. Unknown ids, ids of
+     * other jobs and internal-only rows all resolve to null, which callers treat
+     * as "no-op" — never as an error.
+     */
+    private resolveAckTarget;
+    private durableProjectedPosition;
+}
+//# sourceMappingURL=JobEventStream.d.ts.map