pixivflow 3.1.0 → 3.3.0

package/dist/scheduler/JobEventStream.js

@@ -0,0 +1,251 @@
+"use strict";
+/**
+ * The durable job event stream (Workflow Protocol v1 §events).
+ *
+ * The stream is a PROJECTION of `delivery_events` — the log the delivery
+ * subsystem already writes — plus ONE durable ack cursor
+ * (`job_event_cursors`). There is no second event queue and no second table
+ * family: `slot_id` is the job id, `ts` is the protocol `at`, and `event` is the
+ * internal kind translated by the single mapping in `JobFacade`.
+ *
+ * Three properties this module owns, because they are protocol obligations:
+ *
+ *  1. **A terminal job always has a terminal event.** `SlotCoordinator`'s
+ *     `execution.summary` hook only fires for success/partial/failed, and
+ *     cancelled/expired slots would never produce one. Instead of widening the
+ *     slot/business-status logic, the lifecycle events are *reconciled from the
+ *     durable projection*: whatever the ledger already implies must exist, is
+ *     materialised exactly once (mirroring `reconcileScheduleSummaries`). The
+ *     page is therefore never empty for a job that exists.
+ *
+ *  2. **Ack is monotonic, idempotent and side-effect free.** It writes one row
+ *     in `job_event_cursors` and nothing else. An unknown or older cursor is a
+ *     no-op that reports the durable state — never an error.
+ *
+ *  3. **A declared `callback_url` is never silently dropped.** The event row and
+ *     its callback intent are committed in ONE transaction, and a sweep over
+ *     durable rows repairs any job whose terminal event is still missing (for
+ *     example because the process died mid-flight). Retry/dead-lettering stay in
+ *     the existing outbox worker.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.JobEventStream = exports.EVENT_CALLBACK_MAX_ATTEMPTS = exports.EVENTS_PAGE_LIMIT = void 0;
+const logger_1 = require("../logger");
+const JobFacade_1 = require("./JobFacade");
+const JobView_1 = require("./JobView");
+const ProtocolErrors_1 = require("./ProtocolErrors");
+/** One page of events. Bounded so a long-running job cannot stream unbounded JSON. */
+exports.EVENTS_PAGE_LIMIT = 200;
+/** Bounded callback budget: a flaky third-party URL must converge, not retry forever. */
+exports.EVENT_CALLBACK_MAX_ATTEMPTS = 8;
+/** How many callback-bearing jobs one reconcile sweep repairs. */
+const RECONCILE_SWEEP_LIMIT = 50;
+class JobEventStream {
+    deps;
+    constructor(deps) {
+        this.deps = deps;
+    }
+    get db() {
+        return this.deps.database;
+    }
+    /**
+     * `$defs/EventPage`. Throws 404 for an unknown job, exactly like
+     * `GET /jobs/:jobId`.
+     */
+    page(jobId, query) {
+        const job = this.requireJob(jobId);
+        // The ledger, not a timer, decides which lifecycle events exist.
+        this.reconcile(jobId);
+        const kinds = (0, JobFacade_1.projectedInternalKinds)();
+        const cursor = this.db.outbox.jobEventCursor(jobId);
+        const unackedPosition = cursor ? { at: cursor.at, rowId: cursor.rowId } : null;
+        const after = query.unackedOnly
+            ? unackedPosition
+            : this.resolveAfter(jobId, query.after);
+        const rows = this.db.outbox.listSlotEvents(jobId, kinds, after, exports.EVENTS_PAGE_LIMIT + 1);
+        const hasMore = rows.length > exports.EVENTS_PAGE_LIMIT;
+        const pageRows = hasMore ? rows.slice(0, exports.EVENTS_PAGE_LIMIT) : rows;
+        return (0, JobFacade_1.buildEventPage)({
+            jobId,
+            sources: pageRows.map(toSource),
+            job,
+            correlationId: job.correlation_id ?? null,
+            hasMore,
+            // `unacked` always counts from the durable cursor, never from `after`:
+            // a client paging backwards must not see the count move.
+            unacked: this.db.outbox.countSlotEvents(jobId, kinds, unackedPosition),
+            serverTime: this.deps.now(),
+        });
+    }
+    /**
+     * `$defs/AckResult`. `ack_through` is monotonic and idempotent: replaying it,
+     * or sending an unknown/older cursor, is a no-op that reports what is durably
+     * stored. Acking never touches the slot ledger.
+     */
+    ack(jobId, body) {
+        const { ackThrough } = (0, JobFacade_1.parseAckBody)(body);
+        this.requireJob(jobId);
+        this.reconcile(jobId);
+        const kinds = (0, JobFacade_1.projectedInternalKinds)();
+        const position = this.resolveAckTarget(jobId, ackThrough);
+        if (position)
+            this.db.outbox.advanceJobEventCursor(jobId, position, this.deps.now());
+        const cursor = this.db.outbox.jobEventCursor(jobId);
+        const unackedPosition = cursor ? { at: cursor.at, rowId: cursor.rowId } : null;
+        const unacked = this.db.outbox.countSlotEvents(jobId, kinds, unackedPosition);
+        const total = this.db.outbox.countSlotEvents(jobId, kinds, null);
+        return (0, JobFacade_1.buildAckResult)({
+            jobId,
+            acked: total - unacked,
+            unacked,
+            serverTime: this.deps.now(),
+        });
+    }
+    /**
+     * Materialise every lifecycle event the durable ledger already implies, at
+     * most once each, and enqueue the matching callback in the same transaction.
+     * `callbackUrl` is only read when THIS call is the one that first records the
+     * job's request: a replay never rewrites the declared callback.
+     */
+    reconcile(jobId, callbackUrl = null) {
+        const job = this.requireJob(jobId);
+        this.ensure(job.job_id, job, 'job.requested', job.created_at, callbackUrl);
+        if (job.started_at !== undefined && job.started_at !== null) {
+            this.ensure(job.job_id, job, 'job.execution_started', job.started_at, null);
+        }
+        const terminal = (0, JobFacade_1.terminalJobEventKind)(job.status);
+        if (terminal)
+            this.ensure(job.job_id, job, terminal, job.updated_at, null);
+    }
+    /**
+     * Repair sweep for jobs that declared a callback and have no durable terminal
+     * event yet, so a process that died before the terminal reconcile still
+     * delivers its terminal callback without waiting for a poll. Only jobs that
+     * already own a durable `job.requested` row are in scope: that is exactly the
+     * set whose callback endpoint is known, and re-running it can never queue a
+     * second copy (the event_id is the outbox idempotency key). A job that never
+     * recorded its request event is repaired on the consumer's next read instead.
+     * Returns how many jobs it looked at, so a caller can assert it drains.
+     */
+    reconcileOutstanding(limit = RECONCILE_SWEEP_LIMIT) {
+        const slotIds = this.db.outbox.slotsAwaitingTerminalEvent(JobFacade_1.TERMINAL_JOB_EVENT_KINDS, limit);
+        for (const slotId of slotIds) {
+            try {
+                this.reconcile(slotId);
+            }
+            catch (error) {
+                // One broken job must never stop the outbox pump.
+                logger_1.logger.warn('Job event reconcile failed', { slotId, error });
+            }
+        }
+        return slotIds.length;
+    }
+    ensure(jobId, job, kind, at, callbackUrl) {
+        if (!job)
+            return;
+        // One commit: "the event is durable" and "the callback is owed" cannot
+        // diverge, so a crash can never lose a declared callback.
+        this.db.transaction(() => {
+            const write = this.db.outbox.recordSlotEventOnce({
+                slotId: jobId,
+                event: kind,
+                at,
+                deliveryTarget: callbackUrl,
+            });
+            if (!write.inserted)
+                return;
+            // Read the callback back from the durable declaration rather than from the
+            // argument, so the terminal event and the request event can never name
+            // different endpoints.
+            const declared = this.declaredCallbackUrl(jobId);
+            if (!declared)
+                return;
+            this.enqueueCallback(job, write.event, declared);
+        });
+    }
+    /** The `callback_url` a job declared, read from its durable request event. */
+    declaredCallbackUrl(jobId) {
+        return this.db.outbox.slotEvent(jobId, 'job.requested')?.deliveryTarget ?? null;
+    }
+    enqueueCallback(job, row, url) {
+        const payload = {
+            event: null,
+            job_id: job.job_id,
+            context: { slotId: job.job_id },
+        };
+        // The body is built by the SAME builder the endpoint serves, so a callback
+        // can never carry a shape the reader would not produce.
+        const body = this.eventBodyFor(job, row);
+        if (!body)
+            return;
+        payload.event = body;
+        this.db.outbox.enqueue({
+            kind: 'event_callback',
+            deliveryTarget: url,
+            // Reuses `idx_outbox_key`: replaying an admission, or re-running the
+            // sweep, can never queue a second copy of the same event.
+            idempotencyKey: `job-event:${job.job_id}:${body.event_id}`,
+            payload,
+            maxAttempts: exports.EVENT_CALLBACK_MAX_ATTEMPTS,
+        });
+    }
+    eventBodyFor(job, row) {
+        const page = (0, JobFacade_1.buildEventPage)({
+            jobId: job.job_id,
+            sources: [toSource(row)],
+            job: this.requireJob(job.job_id),
+            correlationId: job.correlation_id ?? null,
+            hasMore: false,
+            unacked: 0,
+            serverTime: this.deps.now(),
+        });
+        return page.events[0] ?? null;
+    }
+    requireJob(jobId) {
+        const job = (0, JobView_1.protocolJobBySlotId)(this.deps, jobId);
+        if (!job) {
+            throw new ProtocolErrors_1.ProtocolRequestError('invalid_params', 404, { message: 'unknown job' });
+        }
+        return job;
+    }
+    /**
+     * `?after=` is a "give me everything after this" hint. An unrecognised or
+     * foreign cursor cannot filter, so it is treated as absent and the consumer
+     * sees the whole stream again — `event_id` is the dedupe key, and at-least-once
+     * is the contract.
+     */
+    resolveAfter(jobId, after) {
+        if (!after)
+            return null;
+        const position = (0, JobFacade_1.parseProtocolEventId)(after);
+        if (!position)
+            return null;
+        return this.durableProjectedPosition(jobId, position);
+    }
+    /**
+     * Map an opaque `event_id` back to a durable position. Unknown ids, ids of
+     * other jobs and internal-only rows all resolve to null, which callers treat
+     * as "no-op" — never as an error.
+     */
+    resolveAckTarget(jobId, ackThrough) {
+        const position = (0, JobFacade_1.parseProtocolEventId)(ackThrough);
+        if (!position)
+            return null;
+        return this.durableProjectedPosition(jobId, position);
+    }
+    durableProjectedPosition(jobId, position) {
+        const row = this.db.outbox.slotEventById(jobId, position.rowId);
+        if (!row)
+            return null;
+        // Only an event the consumer can actually see may move a cursor: acking an
+        // internal-only row would silently mark projectable events as read.
+        if (!(0, JobFacade_1.isProjectedInternalKind)(row.event))
+            return null;
+        return { at: row.ts, rowId: row.id };
+    }
+}
+exports.JobEventStream = JobEventStream;
+function toSource(row) {
+    return { rowId: row.id, at: row.ts, internalKind: row.event };
+}
+//# sourceMappingURL=JobEventStream.js.map

package/dist/scheduler/JobFacade.d.ts

@@ -0,0 +1,298 @@
+/**
+ * The producer-side projection layer of Workflow Protocol v1 (§11.1).
+ *
+ * Everything a consumer sees about a job is built here from the SAME durable
+ * ledger the legacy endpoints read: no second state store, no parallel status
+ * machine, no consumer vocabulary. The projection deliberately speaks only
+ * protocol words — `job_id`, `status`, `progress`, `error` — while the internal
+ * `TerminalReasonCode` is preserved inside `error.detail.internal_code` for
+ * diagnosis and mapped to the closed protocol code at this single site.
+ *
+ * `GET /capabilities` is likewise derived from the live configuration, so a
+ * config change (budget, deadline ceiling) is reflected without a code change.
+ */
+import { StandaloneConfig } from '../config';
+import { CandidateSearchParams } from './CandidateSearchParams';
+import { JobStatusProjection } from './JobProjection';
+import { ProtocolErrorBody } from './ProtocolErrors';
+import { CandidateSupplyReport } from './TargetOutcome';
+/** The only job type this producer serves today. */
+export declare const JOB_TYPE_CANDIDATE_SEARCH = "candidate_search";
+/** `$defs/ProtocolVersion` — this producer speaks exactly one version. */
+export declare const PROTOCOL_VERSIONS: readonly string[];
+/** `$defs/Job.status`. */
+export type ProtocolJobStatus = 'queued' | 'running' | 'succeeded' | 'failed' | 'cancelled' | 'expired';
+/** `$defs/Progress`: `stage` is required, everything else is optional. */
+export interface ProtocolJobProgress {
+    stage: string;
+    message?: string;
+    at: number;
+}
+/**
+ * `$defs/Job`, as this producer emits it.
+ *
+ * Field names are the contract and are asserted against the vendored schema by
+ * `src/__tests__/protocol/job-facade.test.ts`. Note there is no `slot*`, no
+ * `refetch*` and no consumer-specific key here, by design (§8).
+ */
+export interface ProtocolJob {
+    protocol_version: string;
+    job_id: string;
+    job_type: string;
+    status: ProtocolJobStatus;
+    idempotency_key?: string;
+    correlation_id?: string;
+    created_at: number;
+    updated_at: number;
+    started_at?: number | null;
+    heartbeat_at?: number | null;
+    deadline_at?: number | null;
+    lease_active: boolean;
+    lease_expires_at?: number | null;
+    attempt?: number;
+    progress: ProtocolJobProgress;
+    result?: Record<string, unknown>;
+    /** Always built by `protocolErrorBody`, so the code stays in the closed enum. */
+    error?: ProtocolErrorBody;
+    events_url?: string;
+}
+/** `$defs/JobTypeDeclaration`. */
+export interface ProtocolJobTypeDeclaration {
+    name: string;
+    params_schema: string;
+    result_schema: string;
+    features?: string[];
+    queued_timeout_ms?: number;
+    stall_timeout_ms?: number;
+    heartbeat_interval_ms?: number;
+    default_deadline_ms?: number;
+}
+/** `$defs/Capabilities`. */
+export interface ProtocolCapabilities {
+    protocol_versions: string[];
+    job_types: ProtocolJobTypeDeclaration[];
+    server_time?: number;
+}
+/** `$defs/Candidate`, built from the work this job actually delivered. */
+export interface ProtocolDeliveredWork {
+    workId: string | null;
+    workType: string | null;
+}
+export interface ProtocolJobInput {
+    jobType?: string;
+    /** The shared ledger projection (`buildJobProjection`). */
+    projection: JobStatusProjection;
+    /** The execution ceiling this deployment enforces for the job, if known. */
+    deadlineAt?: number | null;
+    /** The single work item a succeeded job delivered. */
+    delivered?: ProtocolDeliveredWork | null;
+    /** The persisted candidate-supply funnel (counts only, by design). */
+    supply?: CandidateSupplyReport | null;
+    /** Injected clock, for tests. */
+    now?: number;
+}
+/**
+ * Classify the durable cell state.
+ *
+ * The CELL is the truth: it is the unit that executes and the unit the manual
+ * surface submits. Budget verdicts (`queued_too_long`, `stalled_no_heartbeat`,
+ * `execution_timeout`) end as protocol `expired` per §4 — they describe a lost
+ * liveness, not a Pixiv failure — while a consumer cancel is `cancelled`.
+ */
+export declare function protocolJobStatus(projection: JobStatusProjection): ProtocolJobStatus;
+/** The protocol-visible stage label for a job status (§2 `Progress.stage`). */
+export declare function protocolProgressStage(status: ProtocolJobStatus, cellStatus: string): string;
+/**
+ * Build the `$defs/Job` body for one job.
+ *
+ * A terminal job always carries `result` or `error` (§4). The result is built
+ * from what the ledger actually knows — the delivered work item and the
+ * persisted supply counts — never from a re-derived guestimate: `filtered` is
+ * omitted because per-work rejection reasons are not persisted (only counts).
+ */
+export declare function buildProtocolJob(input: ProtocolJobInput): ProtocolJob;
+/**
+ * The execution ceiling this deployment enforces for one schedule's work:
+ * the plan's configured `timeout`, else the scheduler-wide default.
+ */
+export declare function resolveDeadlineMs(config: StandaloneConfig, scheduleId: string): number;
+/**
+ * The declared `default_deadline_ms`: the widest ceiling any enabled plan
+ * grants, so the declaration never promises more time than config allows, and
+ * never less than a plan that does grant it.
+ */
+export declare function resolveDefaultDeadlineMs(config: StandaloneConfig): number;
+/** A `$defs/Task` this producer accepted, normalized for the admission. */
+export interface ParsedTask {
+    idempotencyKey: string;
+    correlationId?: string;
+    /** `params.source.account` — the Pixiv resource identity to assert. */
+    account?: string;
+    /** `params` — the occurrence-scoped retrieval view. */
+    params: CandidateSearchParams;
+    /**
+     * Validated but not enforced in this phase: there is no durable column for a
+     * consumer-supplied ceiling yet, so `deadline_at` reports the ceiling the
+     * producer actually enforces (see `resolveDeadlineMs`) instead of promising
+     * one it would not honour.
+     */
+    deadlineMs?: number;
+    /**
+     * Validated and accepted. Carried durably by the `delivery_events` row for
+     * this Task's `job.requested` event and delivered by the outbox
+     * (`event_callback` rows) — `src/scheduler/JobEventStream.ts`.
+     */
+    callbackUrl?: string;
+}
+/**
+ * Parse and validate a `$defs/Task` body (§2/§3).
+ *
+ * Strict about the documented fields — a wrong type or an out-of-range value is
+ * a 400 `invalid_params`, never a silently coerced default — and deliberately
+ * tolerant about unknown keys, which §3 requires to be ignored so a newer
+ * consumer can talk to an older producer.
+ *
+ * `protocol_version` is judged by kind: absent is a malformed request
+ * (`invalid_params`), present-but-different is a version mismatch
+ * (`unsupported_protocol_version`). `job_type` uses `invalid_params`; the
+ * published error enum has no `unsupported_job_type` member and this producer
+ * does not invent protocol codes.
+ */
+export declare function parseTaskBody(body: unknown): ParsedTask;
+/**
+ * `$defs/Capabilities`, derived from the live configuration: change a budget in
+ * config and this declaration follows without a code change.
+ */
+export declare function buildCapabilities(config: StandaloneConfig, now?: number): ProtocolCapabilities;
+export declare const PROTOCOL_EVENT_TYPES: readonly ["job.accepted", "job.started", "job.progress", "job.succeeded", "job.failed", "job.expired", "job.cancelled"];
+export type ProtocolEventType = (typeof PROTOCOL_EVENT_TYPES)[number];
+/**
+ * The internal job-lifecycle kinds PixivFlow persists for a job. They are
+ * deliberately NOT spelled like the protocol enum: the translation is a
+ * decision made here, never a string passthrough.
+ */
+export declare const JOB_EVENT_KINDS: readonly ["job.requested", "job.execution_started", "job.progressed", "job.outcome_succeeded", "job.outcome_failed", "job.outcome_expired", "job.outcome_cancelled"];
+export type JobEventKind = (typeof JOB_EVENT_KINDS)[number];
+export declare const TERMINAL_JOB_EVENT_KINDS: readonly JobEventKind[];
+/** The internal kind that materialises the terminal event of a finished job. */
+export declare function terminalJobEventKind(status: ProtocolJobStatus): JobEventKind | null;
+/**
+ * The ONE mapping from durable internal kinds to protocol event types.
+ *
+ * `null` means internal-only: the row stays in `delivery_events` for operators
+ * and `runs show`, and never reaches a consumer. Any kind absent from this map
+ * is DROPPED with a warning — an internal kind can never become an unknown
+ * protocol type by accident. The intersection type below makes the mapping
+ * exhaustive over `JobEventKind` at compile time, so adding a new lifecycle kind
+ * without mapping it fails `tsc` instead of shipping.
+ */
+export declare const PROTOCOL_EVENT_FOR_INTERNAL_KIND: Record<string, ProtocolEventType | null> & Record<JobEventKind, ProtocolEventType>;
+/**
+ * The internal kinds a consumer is allowed to see. Callers use this as the SQL
+ * filter so `unacked` can never be pinned by rows that are not projectable.
+ */
+export declare function projectedInternalKinds(): string[];
+/** True when an internal kind is visible to consumers (has a protocol type). */
+export declare function isProjectedInternalKind(internalKind: string): boolean;
+/** Map one internal kind to its protocol type; null when internal-only. */
+export declare function protocolEventTypeFor(internalKind: string): ProtocolEventType | null;
+export interface ProtocolEventPayload {
+    job?: ProtocolJob | null;
+    result?: Record<string, unknown> | null;
+    error?: ProtocolErrorBody | null;
+    /**
+     * Non-normative producer detail. Consumers MUST ignore unknown keys; this is
+     * where internal vocabulary (`internal_kind`) is disclosed without polluting
+     * the protocol fields above.
+     */
+    detail?: Record<string, unknown> | null;
+}
+export interface ProtocolEvent {
+    protocol_version: string;
+    /** Consumers deduplicate on this. Also the ack cursor. */
+    event_id: string;
+    job_id: string;
+    type: ProtocolEventType;
+    at: number;
+    correlation_id?: string;
+    payload?: ProtocolEventPayload;
+}
+export interface ProtocolEventPage {
+    job_id: string;
+    events: ProtocolEvent[];
+    /** Present only when more events remain after this page. */
+    next_after?: string;
+    unacked?: number;
+    server_time?: number;
+}
+export interface ProtocolAckResult {
+    job_id: string;
+    acked: number;
+    unacked: number;
+    server_time?: number;
+}
+/**
+ * A durable event row as the stream sees it. Deliberately a plain shape so the
+ * facade stays independent of the storage layer's row type.
+ */
+export interface ProtocolEventSource {
+    /** `delivery_events.id` — half of the ack cursor identity. */
+    rowId: number;
+    /** `delivery_events.ts` — the honest instant of the transition. */
+    at: number;
+    internalKind: string;
+}
+/** `evt-<at>-<row id>`: opaque, unique, and resolvable back to the row. */
+export declare function protocolEventId(at: number, rowId: number): string;
+/**
+ * A position in a job's durable event log. Structurally identical to the storage
+ * layer's cursor position; declared here so the facade does not depend on the
+ * storage layer's types.
+ */
+export interface ProtocolEventPosition {
+    at: number;
+    rowId: number;
+}
+/** Resolve an opaque event_id to a durable position; null when unrecognised. */
+export declare function parseProtocolEventId(value: string): ProtocolEventPosition | null;
+/** Build one protocol event, or null when the internal kind is internal-only. */
+export declare function buildProtocolEvent(input: {
+    jobId: string;
+    source: ProtocolEventSource;
+    job?: ProtocolJob | null;
+    correlationId?: string | null;
+}): ProtocolEvent | null;
+/**
+ * `$defs/EventPage` from durable rows. `events` is ordered by `at` (then by the
+ * durable row order, which is the same order because writers never move `at`
+ * backwards).
+ */
+export declare function buildEventPage(input: {
+    jobId: string;
+    sources: readonly ProtocolEventSource[];
+    job?: ProtocolJob | null;
+    correlationId?: string | null;
+    hasMore: boolean;
+    unacked: number;
+    serverTime: number;
+}): ProtocolEventPage;
+/** `$defs/AckResult` from the durable cursor. */
+export declare function buildAckResult(input: {
+    jobId: string;
+    acked: number;
+    unacked: number;
+    serverTime: number;
+}): ProtocolAckResult;
+/** `$defs/AckRequest`. */
+export declare function parseAckBody(body: unknown): {
+    ackThrough: string;
+};
+/** Query parameters of `GET /jobs/:jobId/events`. */
+export declare function parseEventQuery(query: {
+    after?: unknown;
+    unacked?: unknown;
+}): {
+    after: string | null;
+    unackedOnly: boolean;
+};
+//# sourceMappingURL=JobFacade.d.ts.map