pixivflow 2.19.5 → 2.20.1

package/dist/scheduler/ScheduleTriggerServer.d.ts

@@ -1,3 +1,4 @@
+import { Request } from 'express';
 import { SlotContext } from './SlotCoordinator';
 import { TriggerSource } from './OccurrenceResolver';
 /**
@@ -51,6 +52,30 @@ export interface TriggerRunResult {
         error?: string | null;
     }>;
 }
+/**
+ * Correlation id issued by the calling clock (`x-schedule-attempt-id`, falling
+ * back to `x-attempt-id`).
+ *
+ * Purely diagnostic: it is trimmed, reduced to `[A-Za-z0-9._:-]` and capped, and
+ * it NEVER participates in occurrence identity. When the clock sends nothing
+ * usable the field is omitted rather than synthesized — a made-up id would make
+ * a missing clock indistinguishable from a satisfied one, which is exactly the
+ * blindness this logging exists to remove.
+ */
+export declare function scheduleAttemptId(req: Request): string | undefined;
+/**
+ * Optional self-identification from the clock that fired: `X-Schedule-Provider`
+ * (e.g. `cron-job-org`, `cloudflare`, `manual`).
+ *
+ * OBSERVABILITY ONLY. It NEVER participates in authorization and NEVER in
+ * occurrence identity: a clock that lies about its name changes one log field
+ * and nothing else. That is deliberate — a self-declared header must not be able
+ * to alter which occurrence runs or whether the request is admitted. Sanitized
+ * and bounded like the attempt id, and omitted when the clock sends nothing
+ * usable rather than defaulted, so "we do not know which clock this was" stays
+ * distinguishable from "the clock identified itself".
+ */
+export declare function scheduleProvider(req: Request): string | undefined;
 export interface TriggerHandlers {
     /** Enabled schedule ids (for 404 on unknown / GET listing). */
     listSchedules(): string[];
@@ -75,16 +100,48 @@ export interface TriggerHandlers {
         retried?: number;
         dead?: number;
     }>;
+    /** Admit one target into a durable, separate manual Slot. */
+    refetch?(targetId: string, requestId: string, correlationId?: string): Promise<{
+        slotId: string;
+        disposition: string;
+    }>;
 }
 export declare class ScheduleTriggerServer {
     private readonly token;
     private readonly handlers;
+    private readonly refetchToken;
     private server;
-    constructor(token: string | undefined, handlers: TriggerHandlers);
+    constructor(token: string | undefined, handlers: TriggerHandlers, refetchToken?: string | undefined);
     /** Token from config or SCHEDULER_TRIGGER_TOKEN env; empty => fail closed. */
     static resolveToken(configured?: string): string | undefined;
     start(host: string, port: number): void;
+    /**
+     * Per-request correlation, installed before every route. It records only the
+     * sanitized attempt id and the arrival instant, so each later line can report
+     * the same `attempt_id` and a handler-relative `elapsed_ms`.
+     */
+    private correlate;
+    /**
+     * `trigger.received` — the clock arrived. Runs before auth, so it carries only
+     * non-sensitive request metadata: never the Authorization header, the bearer
+     * value, the configured token, or any fragment of them.
+     */
+    private received;
+    /**
+     * The single writer for every post-resolution `trigger.*` line. `event` and
+     * `http_status` are passed in from the caller's outcome table, so the line
+     * always describes the same status the response carried.
+     */
+    private triggerOutcome;
+    /**
+     * The non-sensitive correlation fields shared by every `schedule.trigger_*`
+     * line. `attempt_id` and `provider` are OMITTED (never defaulted) when the
+     * clock did not send them, so an unidentified clock stays visible as such.
+     */
+    private attemptMeta;
     private auth;
+    private refetchAuth;
+    private authenticate;
     stop(): void;
 }
 //# sourceMappingURL=ScheduleTriggerServer.d.ts.map

package/dist/scheduler/ScheduleTriggerServer.js

@@ -4,16 +4,89 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ScheduleTriggerServer = void 0;
+exports.scheduleAttemptId = scheduleAttemptId;
+exports.scheduleProvider = scheduleProvider;
 const express_1 = __importDefault(require("express"));
 const node_crypto_1 = require("node:crypto");
 const logger_1 = require("../logger");
+const TRIGGER_OUTCOMES = {
+    accepted: { event: 'schedule.trigger_accepted', httpStatus: 202, status: 'accepted', note: 'queued' },
+    already_running: {
+        event: 'schedule.trigger_already_running',
+        httpStatus: 202,
+        status: 'running',
+        note: 'already_running',
+    },
+    already_completed: {
+        event: 'schedule.trigger_already_completed',
+        httpStatus: 200,
+        status: 'completed',
+        note: 'already_completed',
+    },
+    rejected: { event: 'schedule.trigger_rejected', httpStatus: 503, status: 'rejected', note: 'rejected' },
+};
+/** Bounded, log-safe correlation token. */
+const ATTEMPT_ID_MAX_LENGTH = 64;
+const ATTEMPT_ID_UNSAFE = /[^A-Za-z0-9._:-]/g;
+/**
+ * Correlation id issued by the calling clock (`x-schedule-attempt-id`, falling
+ * back to `x-attempt-id`).
+ *
+ * Purely diagnostic: it is trimmed, reduced to `[A-Za-z0-9._:-]` and capped, and
+ * it NEVER participates in occurrence identity. When the clock sends nothing
+ * usable the field is omitted rather than synthesized — a made-up id would make
+ * a missing clock indistinguishable from a satisfied one, which is exactly the
+ * blindness this logging exists to remove.
+ */
+function scheduleAttemptId(req) {
+    const raw = req.headers['x-schedule-attempt-id'] ?? req.headers['x-attempt-id'];
+    const value = Array.isArray(raw) ? raw[0] : raw;
+    if (typeof value !== 'string')
+        return undefined;
+    const cleaned = value.trim().replace(ATTEMPT_ID_UNSAFE, '').slice(0, ATTEMPT_ID_MAX_LENGTH);
+    return cleaned.length > 0 ? cleaned : undefined;
+}
+/** Max length and character set of the provider tag, which is log-only metadata. */
+const PROVIDER_MAX_LENGTH = 32;
+const PROVIDER_UNSAFE = /[^A-Za-z0-9._-]/g;
+/**
+ * Optional self-identification from the clock that fired: `X-Schedule-Provider`
+ * (e.g. `cron-job-org`, `cloudflare`, `manual`).
+ *
+ * OBSERVABILITY ONLY. It NEVER participates in authorization and NEVER in
+ * occurrence identity: a clock that lies about its name changes one log field
+ * and nothing else. That is deliberate — a self-declared header must not be able
+ * to alter which occurrence runs or whether the request is admitted. Sanitized
+ * and bounded like the attempt id, and omitted when the clock sends nothing
+ * usable rather than defaulted, so "we do not know which clock this was" stays
+ * distinguishable from "the clock identified itself".
+ */
+function scheduleProvider(req) {
+    const raw = req.headers['x-schedule-provider'];
+    const value = Array.isArray(raw) ? raw[0] : raw;
+    if (typeof value !== 'string')
+        return undefined;
+    const cleaned = value.trim().replace(PROVIDER_UNSAFE, '').slice(0, PROVIDER_MAX_LENGTH);
+    return cleaned.length > 0 ? cleaned : undefined;
+}
+/** Epoch ms -> ISO 8601 UTC. Undefined when the value is not a usable instant. */
+function isoUtc(epochMs) {
+    if (typeof epochMs !== 'number' || !Number.isFinite(epochMs))
+        return undefined;
+    const date = new Date(epochMs);
+    // `toISOString` throws on an out-of-range Date, and inside the handler that
+    // would turn a real 202 into a 500 — logging must never move the status code.
+    return Number.isNaN(date.getTime()) ? undefined : date.toISOString();
+}
 class ScheduleTriggerServer {
     token;
     handlers;
+    refetchToken;
     server = null;
-    constructor(token, handlers) {
+    constructor(token, handlers, refetchToken = process.env.PIXIVFLOW_REFETCH_TOKEN?.trim() || undefined) {
         this.token = token;
         this.handlers = handlers;
+        this.refetchToken = refetchToken;
     }
     /** Token from config or SCHEDULER_TRIGGER_TOKEN env; empty => fail closed. */
     static resolveToken(configured) {
@@ -22,6 +95,8 @@ class ScheduleTriggerServer {
     start(host, port) {
         const app = (0, express_1.default)();
         app.use(express_1.default.json());
+        // Correlation + clock start for every route. Cheap enough to be unconditional.
+        app.use(this.correlate);
         app.get('/health', (_req, res) => {
             res.json({ status: 'ok', service: 'pixivflow-scheduler-trigger' });
         });
@@ -32,11 +107,17 @@ class ScheduleTriggerServer {
         });
         // Trigger one schedule by id. The server resolves the occurrence from the
         // schedule cron; the body carries no date and cannot back-fill history.
-        app.post('/internal/schedules/:scheduleId/run', this.auth, async (req, res) => {
+        //
+        // `received` runs BEFORE auth so a trigger that is never admitted (bad token,
+        // disabled endpoint) still leaves durable evidence that the clock fired —
+        // "the clock never arrived" and "the clock was rejected" used to look the
+        // same from outside: nothing at all.
+        app.post('/internal/schedules/:scheduleId/run', this.received, this.auth, async (req, res) => {
             try {
                 const scheduleId = req.params.scheduleId;
                 if (!this.handlers.listSchedules().includes(scheduleId)) {
                     res.status(404).json({ status: 'error', error: `unknown schedule: ${scheduleId}` });
+                    this.triggerOutcome('schedule.trigger_not_found', 404, req, res, { schedule_id: scheduleId });
                     return;
                 }
                 // Optional human label for provenance (e.g. a deploy-layer "今日早班").
@@ -44,56 +125,79 @@ class ScheduleTriggerServer {
                 const label = typeof req.body?.label === 'string' ? req.body.label.slice(0, 80) : undefined;
                 const resolved = this.handlers.resolve(scheduleId, 'http', new Date(), label);
                 if (!('context' in resolved)) {
+                    // A resolve refusal (expired occurrence 410, too-early 425, bad cron
+                    // 400) is a business rejection, so it reports `resolved.status`
+                    // rather than the transport-level 503.
                     res.status(resolved.status).json({ status: 'error', error: resolved.error });
+                    this.triggerOutcome('schedule.trigger_rejected', resolved.status, req, res, {
+                        schedule_id: scheduleId,
+                        disposition: 'rejected',
+                        reason: resolved.error,
+                    });
                     return;
                 }
-                const result = await this.handlers.run(scheduleId, resolved.context);
+                const context = resolved.context;
+                const result = await this.handlers.run(scheduleId, context);
                 // Business disposition, not HTTP luck: an accepted or already-running
                 // occurrence is NOT a completed one. Returning 200/"completed" for a run
                 // that is still executing makes an external clock stop retrying and
                 // silently lose the slot, so 'running' is reported as 202 here.
-                switch (result.disposition) {
-                    case 'already_completed':
-                        res.status(200).json({
-                            status: 'completed',
-                            schedule: result,
-                            note: 'already_completed',
-                        });
-                        return;
-                    case 'accepted':
-                        res.status(202).json({
-                            status: 'accepted',
-                            schedule: result,
-                            note: 'queued',
-                        });
-                        return;
-                    case 'already_running':
-                        res.status(202).json({
-                            status: 'running',
-                            schedule: result,
-                            note: 'already_running',
-                        });
-                        return;
-                    default:
-                        res.status(503).json({
-                            status: 'rejected',
-                            schedule: result,
-                            note: 'rejected',
-                        });
-                        return;
-                }
+                // Status code, body and log event all come from ONE table.
+                const spec = TRIGGER_OUTCOMES[result.disposition] ?? TRIGGER_OUTCOMES.rejected;
+                res.status(spec.httpStatus).json({ status: spec.status, schedule: result, note: spec.note });
+                this.triggerOutcome(spec.event, spec.httpStatus, req, res, {
+                    schedule_id: scheduleId,
+                    slot_id: context.slotId,
+                    occurrence_at: isoUtc(context.occurrenceAt),
+                    disposition: result.disposition,
+                    reason: result.cells?.find((cell) => cell.error)?.error ?? undefined,
+                    trigger_source: context.triggerSource,
+                });
             }
             catch (error) {
                 logger_1.logger.error('Schedule trigger failed', { error: error instanceof Error ? error.message : String(error) });
                 // The slot ledger resumes on the next trigger; a 500 tells the clock to
                 // retry safely (idempotent — the same occurrence/ slot is reused).
                 res.status(500).json({ status: 'error', error: 'schedule run failed; the occurrence will resume on the next trigger' });
+                this.triggerOutcome('schedule.trigger_error', 500, req, res, {
+                    schedule_id: req.params.scheduleId,
+                    reason: error instanceof Error ? error.message : String(error),
+                });
+            }
+        });
+        app.post('/internal/targets/:targetId/refetch', this.refetchAuth, async (req, res) => {
+            const requestId = req.body?.requestId;
+            if (typeof requestId !== 'string' || !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(requestId)) {
+                res.status(400).json({ status: 'error', error: 'requestId must be a UUID' });
+                return;
+            }
+            // Opaque caller correlation (review chain / review id). Optional; bounded
+            // length, never interpreted here. Recorded with the manual Slot so a
+            // recovered worker still correlates the outcome with the requester.
+            const correlationId = req.body?.correlationId;
+            if (correlationId !== undefined && (typeof correlationId !== 'string' || correlationId.length > 200)) {
+                res.status(400).json({ status: 'error', error: 'correlationId must be a string of at most 200 chars' });
+                return;
+            }
+            if (!this.handlers.refetch) {
+                res.status(503).json({ status: 'error', error: 'refetch is unavailable' });
+                return;
+            }
+            try {
+                const result = await this.handlers.refetch(req.params.targetId, requestId, correlationId ?? undefined);
+                res.status(202).json({ status: 'accepted', ...result });
+            }
+            catch (error) {
+                const message = error instanceof Error ? error.message : String(error);
+                const status = message === 'unknown target' ? 404 : message === 'ambiguous target' ? 409 : 500;
+                logger_1.logger.warn('Manual refetch rejected', { targetId: req.params.targetId, requestId, status, error: message });
+                res.status(status).json({ status: 'error', error: status === 500 ? 'refetch admission failed' : message });
             }
         });
         // Convergence endpoint: after a machine stop/start, an operator or an
         // external watcher can ask the process to flush due deliveries/notifications
         // without running candidate selection. Deployment-agnostic (no platform refs).
-        app.post('/internal/outbox/drain', this.auth, async (_req, res) => {
+        app.post('/internal/outbox/drain', this.auth, async (req, res) => {
             try {
                 if (!this.handlers.drainOutbox) {
                     res.status(503).json({ status: 'error', error: 'outbox worker not available in this runtime' });
@@ -103,7 +207,11 @@ class ScheduleTriggerServer {
                 res.json({ status: 'ok', result });
             }
             catch (error) {
-                logger_1.logger.error('Outbox drain failed', { error: error instanceof Error ? error.message : String(error) });
+                // Same correlation as the trigger lines, at zero extra cost.
+                logger_1.logger.error('Outbox drain failed', {
+                    ...this.attemptMeta(req, res),
+                    error: error instanceof Error ? error.message : String(error),
+                });
                 res.status(500).json({ status: 'error', error: 'outbox drain failed; rows remain durable and retry' });
             }
         });
@@ -111,18 +219,76 @@ class ScheduleTriggerServer {
             logger_1.logger.info('Schedule trigger server listening', { host, port, auth: this.token ? 'bearer' : 'DISABLED (no token)' });
         });
     }
+    /**
+     * Per-request correlation, installed before every route. It records only the
+     * sanitized attempt id and the arrival instant, so each later line can report
+     * the same `attempt_id` and a handler-relative `elapsed_ms`.
+     */
+    correlate = (req, res, next) => {
+        res.locals.triggerStartedAt = Date.now();
+        res.locals.triggerAttemptId = scheduleAttemptId(req);
+        res.locals.triggerProvider = scheduleProvider(req);
+        next();
+    };
+    /**
+     * `trigger.received` — the clock arrived. Runs before auth, so it carries only
+     * non-sensitive request metadata: never the Authorization header, the bearer
+     * value, the configured token, or any fragment of them.
+     */
+    received = (req, res, next) => {
+        logger_1.logger.info('Schedule trigger received', { event: 'schedule.trigger_received', ...this.attemptMeta(req, res) });
+        next();
+    };
+    /**
+     * The single writer for every post-resolution `trigger.*` line. `event` and
+     * `http_status` are passed in from the caller's outcome table, so the line
+     * always describes the same status the response carried.
+     */
+    triggerOutcome(event, httpStatus, req, res, extra) {
+        logger_1.logger.info('Schedule trigger outcome', {
+            event,
+            ...this.attemptMeta(req, res),
+            http_status: httpStatus,
+            ...extra,
+        });
+    }
+    /**
+     * The non-sensitive correlation fields shared by every `schedule.trigger_*`
+     * line. `attempt_id` and `provider` are OMITTED (never defaulted) when the
+     * clock did not send them, so an unidentified clock stays visible as such.
+     */
+    attemptMeta(req, res) {
+        const attemptId = res.locals.triggerAttemptId;
+        const provider = res.locals.triggerProvider;
+        const startedAt = res.locals.triggerStartedAt ?? Date.now();
+        return {
+            ...(attemptId ? { attempt_id: attemptId } : {}),
+            ...(provider ? { provider } : {}),
+            path: req.path,
+            method: req.method,
+            elapsed_ms: Date.now() - startedAt,
+        };
+    }
     auth = (req, res, next) => {
-        if (!this.token) {
-            // Fail closed: never allow an unauthenticated trigger in production.
-            res.status(503).json({ status: 'error', error: 'trigger disabled: SCHEDULER_TRIGGER_TOKEN not configured' });
+        this.authenticate(this.token, req, res, next);
+    };
+    refetchAuth = (req, res, next) => {
+        this.authenticate(this.refetchToken, req, res, next);
+    };
+    authenticate(expected, req, res, next) {
+        if (!expected) {
+            // Fail closed: never allow an unauthenticated trigger in production. This
+            // is an admission failure, so it is reported with the status actually sent.
+            res.status(503).json({ status: 'error', error: 'endpoint disabled: token not configured' });
+            this.triggerOutcome('schedule.trigger_unauthorized', 503, req, res, { reason: 'endpoint disabled: no token configured' });
             return;
         }
         const header = req.headers.authorization ?? '';
         const presented = header.startsWith('Bearer ') ? header.slice(7).trim() : '';
-        const expected = this.token;
         let ok = presented.length === expected.length;
         if (ok) {
             try {
+                // Values only, never logged: the timing check is deliberately opaque.
                 ok = (0, node_crypto_1.timingSafeEqual)(Buffer.from(presented), Buffer.from(expected));
             }
             catch {
@@ -131,10 +297,11 @@ class ScheduleTriggerServer {
         }
         if (!ok) {
             res.status(401).json({ status: 'error', error: 'unauthorized' });
+            this.triggerOutcome('schedule.trigger_unauthorized', 401, req, res, { reason: 'invalid or missing bearer token' });
             return;
         }
         next();
-    };
+    }
     stop() {
         this.server?.close();
         this.server = null;

package/dist/scheduler/SlotCoordinator.d.ts

@@ -18,8 +18,8 @@ import { TargetExecutionContext, WorkBinding } from './WorkIdentity';
 export declare const SLOT_LEASE_TTL_MS: number;
 export declare const SLOT_HEARTBEAT_MS: number;
 /**
- * Durable execution context attached to a run. A scheduled occurrence always
- * has a slotId; an ad-hoc/manual run has none (it never touches the slot ledger).
+ * Durable execution context attached to a scheduled occurrence or a remote
+ * manual replacement. The local run-once CLI has no SlotContext.
  */
 export interface SlotContext {
     slotId: string;
@@ -36,6 +36,16 @@ export interface SlotContext {
      */
     slotName: string;
     slotDate: string;
+    /**
+     * Remote manual replacement ("重抓") request UUID. Present only on slots
+     * opened by the authenticated refetch endpoint; null for scheduled
+     * occurrences. Persisted with the slot so a sleeping worker that recovers
+     * the slot still knows it was a manual replacement (delivery correlation,
+     * outcome reporting) without any in-memory state.
+     */
+    manualRequestId?: string;
+    /** Opaque caller correlation (review chain / review id). Null unless manual. */
+    correlationId?: string;
 }
 export interface SlotCellSummary {
     targetId: string;
@@ -50,6 +60,54 @@ export interface SlotRunSummary {
     alreadyCompleted: boolean;
     cells: SlotCellSummary[];
 }
+/** Terminal statuses only: the rollup that ends an occurrence's life. */
+export type ScheduleOutcomeStatus = 'success' | 'partial' | 'failed';
+export interface ScheduleOutcomeTarget {
+    target_id: string;
+    status: CellStatus;
+    work_id: string | null;
+    error: string | null;
+}
+/**
+ * Business categories an operator actually asks about, counted per cell. The
+ * point is to separate "nothing suitable existed" (no_match) and "already
+ * delivered" (duplicate) from real failures: lumping them together made a
+ * healthy run look broken, and a broken one look routine.
+ *
+ * Each cell lands in exactly ONE category, in this precedence:
+ * submitted -> no_match -> duplicate -> delivery_failed -> executor_failed.
+ * `delivery_failed` is checked before `executor_failed` because a terminally
+ * lost delivery ALSO leaves the cell in the `failed` state — counting it twice
+ * would invent a second failure that does not exist.
+ */
+export interface ScheduleOutcomeCells {
+    total: number;
+    submitted: number;
+    no_match: number;
+    duplicate: number;
+    /** True when there is at least one non-submitted cell and ALL of them are duplicates. */
+    all_duplicates: boolean;
+    executor_failed: number;
+    delivery_failed: number;
+    targets: ScheduleOutcomeTarget[];
+}
+/**
+ * The single structured terminal record for one schedule occurrence. The SAME
+ * object is both logged as `schedule.outcome` and persisted into the existing
+ * `execution.summary` detail, so the log line and the durable row cannot
+ * disagree — and no second event name or shadow ledger is introduced.
+ */
+export interface ScheduleOutcomeRecord {
+    event: 'schedule.outcome';
+    schedule_id: string;
+    slot_id: string;
+    occurrence_at: string | undefined;
+    occurrence_date: string;
+    status: ScheduleOutcomeStatus;
+    /** From the slot row's own started_at/completed_at columns, when both exist. */
+    duration_ms: number | undefined;
+    cells: ScheduleOutcomeCells;
+}
 export interface SlotResolveResult {
     context?: SlotContext;
     error?: string;
@@ -196,11 +254,31 @@ export declare class SlotCoordinator {
     releaseRunLease(slotId: string, owner: string): void;
     /** Roll cell results up into the slot status (one place computes the aggregate). */
     finish(slot: SlotContext, schedule: ScheduleConfig, targets: TargetConfig[]): SlotRunSummary;
+    /**
+     * Project the terminal slot row into the one structured outcome record. Every
+     * field comes from durable state — the slot row, its cells, and the delivery
+     * ledger. Nothing is inferred from error-message text.
+     */
+    private scheduleOutcome;
+    /**
+     * One cell -> one business category. `delivery_failed` is derived from the
+     * durable delivery ledger through the SAME port the FSM already uses, never
+     * from an error string, and it is decided BEFORE `executor_failed` because a
+     * terminally lost delivery also leaves the cell `failed`: one cause, one count.
+     */
+    private classifyCell;
     /**
      * Persist a one-row incident/execution summary into delivery_events
      * (event='execution.summary') at the terminal rollup only. Event-row storage
      * reuses the audit table + `runs show` read path instead of inventing a
      * summary table or overloading execution_log's illustration/novel typing.
+     *
+     * The `schedule.outcome` line is emitted HERE, immediately after that durable
+     * write and behind the SAME `hasExecutionSummary` dedupe, because the summary
+     * row is the occurrence-level terminal identity that already exists. Recovery
+     * that re-rolls the same terminal slot therefore logs nothing a second time,
+     * with no shadow ledger and no second event name. The identical object goes to
+     * both sinks, so the line and the row cannot disagree.
      */
     private persistExecutionSummary;
     completedSummary(slotId: string, schedule: ScheduleConfig): SlotRunSummary;