pixivflow 3.1.0 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/README.md +26 -1
  2. package/dist/commands/SchedulerCommand.js +31 -51
  3. package/dist/commands/scheduler-runtime.js +47 -7
  4. package/dist/config/defaults.d.ts +2 -0
  5. package/dist/config/defaults.js +6 -0
  6. package/dist/config/types.d.ts +20 -0
  7. package/dist/config/validation.js +11 -0
  8. package/dist/delivery/DeliveryDispatcher.d.ts +14 -0
  9. package/dist/delivery/DeliveryDispatcher.js +14 -0
  10. package/dist/delivery/EventCallbackDelivery.d.ts +54 -0
  11. package/dist/delivery/EventCallbackDelivery.js +89 -0
  12. package/dist/delivery/OutboxWorker.d.ts +11 -0
  13. package/dist/delivery/OutboxWorker.js +51 -1
  14. package/dist/package.json +1 -1
  15. package/dist/scheduler/CandidateSearchParams.d.ts +76 -0
  16. package/dist/scheduler/CandidateSearchParams.js +94 -0
  17. package/dist/scheduler/JobCancellation.d.ts +36 -0
  18. package/dist/scheduler/JobCancellation.js +73 -0
  19. package/dist/scheduler/JobEventStream.d.ts +94 -0
  20. package/dist/scheduler/JobEventStream.js +251 -0
  21. package/dist/scheduler/JobFacade.d.ts +298 -0
  22. package/dist/scheduler/JobFacade.js +622 -0
  23. package/dist/scheduler/JobProjection.d.ts +67 -0
  24. package/dist/scheduler/JobProjection.js +32 -0
  25. package/dist/scheduler/JobView.d.ts +34 -0
  26. package/dist/scheduler/JobView.js +82 -0
  27. package/dist/scheduler/ManualJobAdmission.d.ts +126 -0
  28. package/dist/scheduler/ManualJobAdmission.js +310 -0
  29. package/dist/scheduler/ManualJobService.d.ts +57 -0
  30. package/dist/scheduler/ManualJobService.js +91 -0
  31. package/dist/scheduler/ManualRefetchAdapter.d.ts +26 -0
  32. package/dist/scheduler/ManualRefetchAdapter.js +41 -0
  33. package/dist/scheduler/MultiScheduleManager.d.ts +17 -0
  34. package/dist/scheduler/MultiScheduleManager.js +68 -6
  35. package/dist/scheduler/ProtocolErrors.d.ts +68 -0
  36. package/dist/scheduler/ProtocolErrors.js +94 -0
  37. package/dist/scheduler/ScheduleTriggerServer.d.ts +56 -7
  38. package/dist/scheduler/ScheduleTriggerServer.js +166 -1
  39. package/dist/scheduler/SlotBusinessStatus.d.ts +5 -1
  40. package/dist/scheduler/SlotBusinessStatus.js +6 -0
  41. package/dist/scheduler/SlotCoordinator.d.ts +49 -5
  42. package/dist/scheduler/SlotCoordinator.js +89 -22
  43. package/dist/scheduler/StallSweep.d.ts +65 -0
  44. package/dist/scheduler/StallSweep.js +105 -0
  45. package/dist/scheduler/TargetOutcome.d.ts +39 -1
  46. package/dist/scheduler/TargetOutcome.js +51 -1
  47. package/dist/scheduler/ledger-time.d.ts +23 -0
  48. package/dist/scheduler/ledger-time.js +37 -0
  49. package/dist/storage/DatabaseMigration.js +48 -0
  50. package/dist/storage/repositories/DeliveryRepository.d.ts +6 -0
  51. package/dist/storage/repositories/DeliveryRepository.js +13 -0
  52. package/dist/storage/repositories/OutboxRepository.d.ts +73 -1
  53. package/dist/storage/repositories/OutboxRepository.js +142 -0
  54. package/dist/storage/repositories/SlotRepository.d.ts +34 -0
  55. package/dist/storage/repositories/SlotRepository.js +61 -2
  56. package/dist/version.js +1 -1
  57. package/dist/webui/package.json +1 -1
  58. package/examples/gateway/README.md +4 -0
  59. package/examples/onebot-adapter/README.md +139 -0
  60. package/examples/onebot-adapter/server.mjs +612 -0
  61. package/package.json +2 -1
@@ -0,0 +1,105 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.STALL_SWEEP_BATCH_LIMIT = exports.DEFAULT_STALL_TIMEOUT_MS = exports.DEFAULT_QUEUED_TIMEOUT_MS = exports.STALL_SWEEP_MIN_TIMEOUT_MS = void 0;
4
+ exports.resolveStallTimeouts = resolveStallTimeouts;
5
+ exports.sweepStalledSlots = sweepStalledSlots;
6
+ const TargetOutcome_1 = require("./TargetOutcome");
7
+ const ledger_time_1 = require("./ledger-time");
8
+ const logger_1 = require("../logger");
9
+ /** Nothing is ever stalled before this: a floor against a misconfigured budget. */
10
+ exports.STALL_SWEEP_MIN_TIMEOUT_MS = 60 * 1000;
11
+ /** Admitted but never claimed for this long -> `queued_too_long`. */
12
+ exports.DEFAULT_QUEUED_TIMEOUT_MS = 30 * 60 * 1000;
13
+ /** No heartbeat/lease for this long while `running` -> `stalled_no_heartbeat`. */
14
+ exports.DEFAULT_STALL_TIMEOUT_MS = 15 * 60 * 1000;
15
+ /** Bounded batch: one tick must stay cheap on a large ledger. */
16
+ exports.STALL_SWEEP_BATCH_LIMIT = 100;
17
+ /** The two budgets as configured, each clamped to the floor / its default. */
18
+ function resolveStallTimeouts(rt) {
19
+ return {
20
+ queuedTimeoutMs: positiveMs(rt?.queuedTimeoutMs, exports.DEFAULT_QUEUED_TIMEOUT_MS),
21
+ stallTimeoutMs: positiveMs(rt?.stallTimeoutMs, exports.DEFAULT_STALL_TIMEOUT_MS),
22
+ };
23
+ }
24
+ function positiveMs(value, fallback) {
25
+ return typeof value === 'number' &&
26
+ Number.isFinite(value) &&
27
+ value >= exports.STALL_SWEEP_MIN_TIMEOUT_MS
28
+ ? Math.trunc(value)
29
+ : fallback;
30
+ }
31
+ /** One bounded, liveness-aware sweep. Never throws for a single bad row. */
32
+ function sweepStalledSlots(database, options = {}) {
33
+ const now = options.now ?? Date.now();
34
+ const limit = options.limit ?? exports.STALL_SWEEP_BATCH_LIMIT;
35
+ const { queuedTimeoutMs, stallTimeoutMs } = resolveStallTimeouts(options);
36
+ const skip = options.skipSlotIds;
37
+ const result = { scanned: 0, queuedTooLong: 0, stalledNoHeartbeat: 0 };
38
+ // Rule 1: admitted but never claimed.
39
+ const queued = database.slots.agedPendingSlots((0, ledger_time_1.sqliteUtcTimestamp)(now - queuedTimeoutMs), now, limit);
40
+ result.scanned += queued.length;
41
+ for (const slot of queued) {
42
+ if (skip?.has(slot.id))
43
+ continue;
44
+ if (terminaliseStalledSlot(database, slot, 'queued_too_long', now - queuedTimeoutMs)) {
45
+ result.queuedTooLong++;
46
+ }
47
+ }
48
+ // Rule 2: claimed, then lost its worker. The lease must be dead AND progress
49
+ // stale; either one alone is a healthy (or merely slow) run, never a stall.
50
+ const stalled = database.slots.stalledRunningSlots((0, ledger_time_1.sqliteUtcTimestamp)(now - stallTimeoutMs), now, limit);
51
+ result.scanned += stalled.length;
52
+ for (const slot of stalled) {
53
+ if (skip?.has(slot.id))
54
+ continue;
55
+ if (terminaliseStalledSlot(database, slot, 'stalled_no_heartbeat', now - stallTimeoutMs)) {
56
+ result.stalledNoHeartbeat++;
57
+ }
58
+ }
59
+ return result;
60
+ }
61
+ /**
62
+ * Terminalise one stalled Slot: every cell that never made progress becomes
63
+ * `failed` with the liveness reason, and the Slot derives its own rollup so a
64
+ * partially delivered occurrence still reports `partial` instead of being
65
+ * misreported as a plain failure. Returns false when nothing was written.
66
+ */
67
+ function terminaliseStalledSlot(database, slot, code, sinceMs) {
68
+ const message = TargetOutcome_1.TERMINAL_REASON_MESSAGES[code];
69
+ const cells = database.slots.getCells(slot.id);
70
+ let failedCells = 0;
71
+ for (const cell of cells) {
72
+ if (cell.status !== 'pending' && cell.status !== 'selected')
73
+ continue;
74
+ try {
75
+ database.slots.transitionCell(slot.id, cell.targetId, 'failed', message);
76
+ database.slots.setCellTerminalReason(slot.id, cell.targetId, code, message);
77
+ failedCells++;
78
+ }
79
+ catch (error) {
80
+ logger_1.logger.debug('Stall sweep skipped a cell it could not terminalise', {
81
+ slot: slot.id,
82
+ target: cell.targetId,
83
+ error: error instanceof Error ? error.message : String(error),
84
+ });
85
+ }
86
+ }
87
+ const derived = cells.length === 0 ? 'failed' : database.slots.deriveSlotStatus(slot.id);
88
+ // A stalled slot must never be reported as healthy: an empty ledger or a
89
+ // rollup that still contains an in-flight delivery cell stays `failed`.
90
+ const status = derived === 'success' || derived === 'running' ? 'failed' : derived;
91
+ database.slots.markSlotStatus(slot.id, status, message);
92
+ logger_1.logger.warn('Terminalising a slot that made no progress within its liveness budget', {
93
+ slot: slot.id,
94
+ schedule: slot.scheduleId,
95
+ trigger_source: slot.triggerSource,
96
+ manual_request_id: slot.manualRequestId,
97
+ correlation_id: slot.correlationId,
98
+ reason_code: code,
99
+ slot_status: slot.status,
100
+ cells_failed: failedCells,
101
+ stalled_since: new Date(sinceMs).toISOString(),
102
+ });
103
+ return true;
104
+ }
105
+ //# sourceMappingURL=StallSweep.js.map
@@ -17,6 +17,7 @@
17
17
  * item after its bounded candidate scan, including the scan bookkeeping so
18
18
  * "completed with nothing done" is impossible to report silently.
19
19
  */
20
+ import type { ProtocolErrorCode } from './ProtocolErrors';
20
21
  export type WorkType = 'illustration' | 'novel';
21
22
  /**
22
23
  * Why ONE candidate work was skipped without producing a business result.
@@ -330,7 +331,28 @@ export type TerminalReasonCode =
330
331
  /** Every candidate the scan saw was already delivered/pending for this target. */
331
332
  | 'duplicate_exhausted'
332
333
  /** Candidates existed but none passed the target's own filters. */
333
- | 'filter_exhausted' | 'download_timeout' | 'download_failed' | 'metadata_failed' | 'rate_limited' | 'auth_failed' | 'remote_http_error' | 'delivery_failed' | 'telepost_rejected' | 'telegram_failed' | 'network_error' | 'execution_timeout' | 'configuration_error' | 'internal_error';
334
+ | 'filter_exhausted' | 'download_timeout' | 'download_failed' | 'metadata_failed' | 'rate_limited' | 'auth_failed' | 'remote_http_error' | 'delivery_failed' | 'telepost_rejected' | 'telegram_failed' | 'network_error' | 'execution_timeout' | 'configuration_error' | 'internal_error'
335
+ /**
336
+ * Liveness verdicts of the stall sweep (§liveness). The Slot was admitted but
337
+ * made no progress within its budget: either nothing ever claimed it
338
+ * (`queued_too_long`) or its owner stopped heartbeating and its lease expired
339
+ * (`stalled_no_heartbeat`). These describe the LEDGER, not a Pixiv failure.
340
+ */
341
+ | 'queued_too_long' | 'stalled_no_heartbeat'
342
+ /**
343
+ * The owning run ended while a delivery intent had no actionable outbox row
344
+ * left (dead-lettered, cancelled by an operator, or lost with the process),
345
+ * so no worker would ever converge that cell.
346
+ */
347
+ | 'delivery_abandoned'
348
+ /**
349
+ * A CONSUMER/operator stopped the work on purpose (§cancel). This is not a
350
+ * Pixiv failure and not a system failure: nothing went wrong, the work is
351
+ * simply no longer wanted, so it must never be alerted on or retried
352
+ * automatically. The cell is terminalised as `failed` (the existing FSM has no
353
+ * separate "cancelled" state and none is invented) with this code on it.
354
+ */
355
+ | 'cancelled_by_consumer';
334
356
  export interface TerminalReason {
335
357
  code: TerminalReasonCode;
336
358
  /** Business-language message an operator/reviewer reads directly. */
@@ -351,6 +373,22 @@ export interface OperationalReason extends TerminalReason {
351
373
  export declare function operationalReasonForCode(code: string, message?: string | null): OperationalReason | null;
352
374
  /** User-facing business messages — never stack traces, paths, SQL or tokens. */
353
375
  export declare const TERMINAL_REASON_MESSAGES: Record<TerminalReasonCode, string>;
376
+ /**
377
+ * The producer→protocol error mapping (`protocol/v1/error-mapping.json`,
378
+ * `producer_internal`), expressed as an exhaustive `Record` so a NEW internal
379
+ * code cannot compile without a protocol mapping: an unmapped internal code
380
+ * would otherwise leak this service's private vocabulary to a consumer.
381
+ *
382
+ * The translation happens ONCE, at the job facade. The legacy manual endpoints
383
+ * keep reporting internal codes (diagnosis), and the generic surface reports the
384
+ * protocol code plus `detail.internal_code`.
385
+ */
386
+ export declare const PROTOCOL_ERROR_CODE_BY_TERMINAL_REASON: Record<TerminalReasonCode, ProtocolErrorCode>;
387
+ /**
388
+ * Protocol error code for a persisted internal terminal reason, or null when the
389
+ * code is absent/unknown (never guess a protocol code for an unrecognised one).
390
+ */
391
+ export declare function protocolErrorCodeForTerminalReason(code: string | null | undefined): ProtocolErrorCode | null;
354
392
  /**
355
393
  * Map a typed terminal TargetOutcome onto the normalized reason. Returns null
356
394
  * for non-terminal outcomes (they have no terminal reason yet).
@@ -1,6 +1,6 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.TERMINAL_REASON_MESSAGES = void 0;
3
+ exports.PROTOCOL_ERROR_CODE_BY_TERMINAL_REASON = exports.TERMINAL_REASON_MESSAGES = void 0;
4
4
  exports.emptyCandidateSupplyReport = emptyCandidateSupplyReport;
5
5
  exports.mergeCandidateSupplyReports = mergeCandidateSupplyReports;
6
6
  exports.withScanSkips = withScanSkips;
@@ -15,6 +15,7 @@ exports.mergeScanSummaries = mergeScanSummaries;
15
15
  exports.classifyCandidateFailure = classifyCandidateFailure;
16
16
  exports.classifyJobLevelOutage = classifyJobLevelOutage;
17
17
  exports.operationalReasonForCode = operationalReasonForCode;
18
+ exports.protocolErrorCodeForTerminalReason = protocolErrorCodeForTerminalReason;
18
19
  exports.terminalReasonFor = terminalReasonFor;
19
20
  /**
20
21
  * A freshly-harvested upstream funnel with no candidates selected yet.
@@ -306,6 +307,10 @@ const OPERATIONAL_REASON_POLICY = {
306
307
  execution_timeout: { stage: 'execution', retryable: true, operatorHint: '检查执行耗时和资源使用后重试。' },
307
308
  configuration_error: { stage: 'configuration', retryable: false, operatorHint: '修正配置并完成校验后再重试。' },
308
309
  internal_error: { stage: 'execution', retryable: false, operatorHint: '查看对应执行记录;若重复发生,请提交诊断信息。' },
310
+ queued_too_long: { stage: 'execution', retryable: true, operatorHint: '队列长时间未开始执行:检查调度器是否在运行/被 resource 队列阻塞后重试。' },
311
+ stalled_no_heartbeat: { stage: 'execution', retryable: true, operatorHint: '执行中途失去心跳(进程中断或卡死):重启后可重试。' },
312
+ delivery_abandoned: { stage: 'delivery', retryable: true, operatorHint: '投递意图已无重试队列(被拒绝或取消):检查接收端后重新重抓。' },
313
+ cancelled_by_consumer: { stage: 'execution', retryable: false, operatorHint: '用户已取消,无需重试。' },
309
314
  };
310
315
  function operationalReasonForCode(code, message) {
311
316
  if (!(code in OPERATIONAL_REASON_POLICY))
@@ -335,7 +340,52 @@ exports.TERMINAL_REASON_MESSAGES = {
335
340
  execution_timeout: '执行超时',
336
341
  configuration_error: '配置错误',
337
342
  internal_error: '内部错误',
343
+ queued_too_long: '排队超时,未能开始执行',
344
+ stalled_no_heartbeat: '执行中断,长时间没有进展',
345
+ delivery_abandoned: '投稿未被处理,已放弃',
346
+ cancelled_by_consumer: '任务已被取消',
338
347
  };
348
+ /**
349
+ * The producer→protocol error mapping (`protocol/v1/error-mapping.json`,
350
+ * `producer_internal`), expressed as an exhaustive `Record` so a NEW internal
351
+ * code cannot compile without a protocol mapping: an unmapped internal code
352
+ * would otherwise leak this service's private vocabulary to a consumer.
353
+ *
354
+ * The translation happens ONCE, at the job facade. The legacy manual endpoints
355
+ * keep reporting internal codes (diagnosis), and the generic surface reports the
356
+ * protocol code plus `detail.internal_code`.
357
+ */
358
+ exports.PROTOCOL_ERROR_CODE_BY_TERMINAL_REASON = {
359
+ no_candidate: 'no_candidate',
360
+ duplicate_exhausted: 'no_candidate',
361
+ filter_exhausted: 'no_candidate',
362
+ download_timeout: 'source_error',
363
+ download_failed: 'source_error',
364
+ metadata_failed: 'source_error',
365
+ rate_limited: 'quota_exceeded',
366
+ auth_failed: 'auth_error',
367
+ remote_http_error: 'source_error',
368
+ delivery_failed: 'delivery_failed',
369
+ telepost_rejected: 'delivery_rejected',
370
+ telegram_failed: 'delivery_failed',
371
+ network_error: 'source_error',
372
+ execution_timeout: 'deadline_exceeded',
373
+ configuration_error: 'internal_error',
374
+ internal_error: 'internal_error',
375
+ queued_too_long: 'queued_too_long',
376
+ stalled_no_heartbeat: 'stalled_no_progress',
377
+ delivery_abandoned: 'delivery_abandoned',
378
+ cancelled_by_consumer: 'cancelled_by_consumer',
379
+ };
380
+ /**
381
+ * Protocol error code for a persisted internal terminal reason, or null when the
382
+ * code is absent/unknown (never guess a protocol code for an unrecognised one).
383
+ */
384
+ function protocolErrorCodeForTerminalReason(code) {
385
+ if (!code)
386
+ return null;
387
+ return exports.PROTOCOL_ERROR_CODE_BY_TERMINAL_REASON[code] ?? null;
388
+ }
339
389
  /**
340
390
  * Map a typed terminal TargetOutcome onto the normalized reason. Returns null
341
391
  * for non-terminal outcomes (they have no terminal reason yet).
@@ -0,0 +1,23 @@
1
+ /**
2
+ * Conversions for the durable ledger's mixed timestamp shapes.
3
+ *
4
+ * `schedule_slots.created_at` / `started_at` / `completed_at` and
5
+ * `schedule_slot_items.created_at` / `updated_at` / `completed_at` come from
6
+ * SQLite `CURRENT_TIMESTAMP`: UTC "YYYY-MM-DD HH:MM:SS" with NO zone marker.
7
+ * `Date.parse` reads that shape as LOCAL time, so the zone is added explicitly
8
+ * instead of being trusted to the engine.
9
+ *
10
+ * `schedule_slots.heartbeat_at` / `lease_until` and the outbox/lease columns are
11
+ * epoch milliseconds written by this process. Both directions live here so a
12
+ * projection and a comparison query can never disagree about which shape a
13
+ * column is in.
14
+ */
15
+ /** SQLite UTC datetime ("YYYY-MM-DD HH:MM:SS") -> epoch ms. */
16
+ export declare function sqliteUtcMs(value: string | null | undefined): number | undefined;
17
+ /**
18
+ * Epoch ms -> the exact string shape SQLite's `CURRENT_TIMESTAMP` writes, so a
19
+ * cutoff can be compared against string columns in SQL instead of loading the
20
+ * whole ledger to filter in JS.
21
+ */
22
+ export declare function sqliteUtcTimestamp(epochMs: number): string;
23
+ //# sourceMappingURL=ledger-time.d.ts.map
@@ -0,0 +1,37 @@
1
+ "use strict";
2
+ /**
3
+ * Conversions for the durable ledger's mixed timestamp shapes.
4
+ *
5
+ * `schedule_slots.created_at` / `started_at` / `completed_at` and
6
+ * `schedule_slot_items.created_at` / `updated_at` / `completed_at` come from
7
+ * SQLite `CURRENT_TIMESTAMP`: UTC "YYYY-MM-DD HH:MM:SS" with NO zone marker.
8
+ * `Date.parse` reads that shape as LOCAL time, so the zone is added explicitly
9
+ * instead of being trusted to the engine.
10
+ *
11
+ * `schedule_slots.heartbeat_at` / `lease_until` and the outbox/lease columns are
12
+ * epoch milliseconds written by this process. Both directions live here so a
13
+ * projection and a comparison query can never disagree about which shape a
14
+ * column is in.
15
+ */
16
+ Object.defineProperty(exports, "__esModule", { value: true });
17
+ exports.sqliteUtcMs = sqliteUtcMs;
18
+ exports.sqliteUtcTimestamp = sqliteUtcTimestamp;
19
+ /** SQLite UTC datetime ("YYYY-MM-DD HH:MM:SS") -> epoch ms. */
20
+ function sqliteUtcMs(value) {
21
+ if (!value)
22
+ return undefined;
23
+ const normalized = /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/.test(value)
24
+ ? `${value.replace(' ', 'T')}Z`
25
+ : value;
26
+ const ms = Date.parse(normalized);
27
+ return Number.isNaN(ms) ? undefined : ms;
28
+ }
29
+ /**
30
+ * Epoch ms -> the exact string shape SQLite's `CURRENT_TIMESTAMP` writes, so a
31
+ * cutoff can be compared against string columns in SQL instead of loading the
32
+ * whole ledger to filter in JS.
33
+ */
34
+ function sqliteUtcTimestamp(epochMs) {
35
+ return new Date(epochMs).toISOString().slice(0, 19).replace('T', ' ');
36
+ }
37
+ //# sourceMappingURL=ledger-time.js.map
@@ -282,6 +282,17 @@ class DatabaseMigration {
282
282
  `CREATE INDEX IF NOT EXISTS idx_system_errors_bot_created ON system_errors(bot_id, created_at)`,
283
283
  `CREATE INDEX IF NOT EXISTS idx_cinventory_pending ON candidate_inventory(status, first_seen_date, expires_at)`,
284
284
  `CREATE INDEX IF NOT EXISTS idx_gateway_connections_type ON gateway_connections(type)`,
285
+ // Durable consumer ack cursor for the Workflow Protocol v1 event stream
286
+ // (§events). One row per job: the newest `delivery_events` row the
287
+ // consumer attests it has persisted. This is a cursor, NOT state — acking
288
+ // never touches the slot ledger, and `unacked` is derived by counting the
289
+ // job's projected events after this position. Monotonic by construction.
290
+ `CREATE TABLE IF NOT EXISTS job_event_cursors (
291
+ job_id TEXT PRIMARY KEY,
292
+ acked_at INTEGER NOT NULL,
293
+ acked_row_id INTEGER NOT NULL,
294
+ updated_at INTEGER NOT NULL
295
+ )`,
285
296
  ];
286
297
  // Phase 1: create tables (idempotent). Must run before any PRAGMA-based
287
298
  // column check, otherwise a fresh DB would report the table as missing and
@@ -310,6 +321,11 @@ class DatabaseMigration {
310
321
  correlation_id: 'ALTER TABLE schedule_slots ADD COLUMN correlation_id TEXT',
311
322
  recovery_request_id: 'ALTER TABLE schedule_slots ADD COLUMN recovery_request_id TEXT',
312
323
  recovery_mode: 'ALTER TABLE schedule_slots ADD COLUMN recovery_mode TEXT',
324
+ // Occurrence-scoped retrieval view of a generic `candidate_search` job
325
+ // (§6). Stored as JSON so a resumed worker re-applies exactly the
326
+ // retrieval the requester asked for; NULL means "as configured", which is
327
+ // every slot written before this column existed.
328
+ params_json: 'ALTER TABLE schedule_slots ADD COLUMN params_json TEXT',
313
329
  };
314
330
  const columnAlters = [];
315
331
  for (const [col, sql] of Object.entries(slotColumnMigrations)) {
@@ -352,6 +368,7 @@ class DatabaseMigration {
352
368
  `CREATE INDEX IF NOT EXISTS idx_delivery_events_execution ON delivery_events(execution_id)`,
353
369
  `CREATE INDEX IF NOT EXISTS idx_delivery_events_outbox ON delivery_events(outbox_id)`,
354
370
  `CREATE INDEX IF NOT EXISTS idx_delivery_events_ts ON delivery_events(ts)`,
371
+ `CREATE INDEX IF NOT EXISTS idx_delivery_events_slot_event ON delivery_events(slot_id, event)`,
355
372
  ];
356
373
  const postMigration = this.db.transaction((stmts) => {
357
374
  for (const sql of stmts) {
@@ -359,6 +376,37 @@ class DatabaseMigration {
359
376
  }
360
377
  });
361
378
  postMigration([...columnAlters, ...indexes]);
379
+ // Manual-work IDENTITY (§identity): `manual_request_id` is the protocol's
380
+ // `idempotency_key`, and one key MUST resolve to exactly one durable slot.
381
+ // Partial unique index, following the `idx_outbox_key` precedent above:
382
+ // every scheduled row is NULL here, so it is unaffected.
383
+ //
384
+ // Guarded on purpose. A ledger written before this migration may hold two
385
+ // rows for one key (the same request id admitted under two plans), and
386
+ // throwing would abort the WHOLE migration and brick startup over
387
+ // historical data. So the conflict is reported loudly and the unique index
388
+ // is skipped; the admission path still refuses to create a second slot, and
389
+ // an operator can resolve the historical rows and restart.
390
+ try {
391
+ const duplicateKeys = this.db
392
+ .prepare(`SELECT manual_request_id AS key, COUNT(*) AS rows FROM schedule_slots
393
+ WHERE manual_request_id IS NOT NULL AND manual_request_id <> ''
394
+ GROUP BY manual_request_id HAVING COUNT(*) > 1 LIMIT 5`)
395
+ .all();
396
+ const manualIdentityIndex = `CREATE UNIQUE INDEX IF NOT EXISTS idx_slots_manual_request ON schedule_slots(manual_request_id) WHERE manual_request_id IS NOT NULL`;
397
+ if (duplicateKeys.length > 0) {
398
+ logger_1.logger.warn('Duplicate manual_request_id rows prevent the unique identity index; deduplicate them and restart', { duplicates: duplicateKeys.map((row) => ({ key: row.key, rows: row.rows })) });
399
+ this.db
400
+ .prepare(`CREATE INDEX IF NOT EXISTS idx_slots_manual_request ON schedule_slots(manual_request_id) WHERE manual_request_id IS NOT NULL`)
401
+ .run();
402
+ }
403
+ else {
404
+ this.db.prepare(manualIdentityIndex).run();
405
+ }
406
+ }
407
+ catch (error) {
408
+ logger_1.logger.warn('Failed to create the manual identity index', { error });
409
+ }
362
410
  // Add is_active column to config_history if it doesn't exist
363
411
  try {
364
412
  // Check if column exists by querying pragma_table_info
@@ -52,6 +52,12 @@ export declare class DeliveryRepository extends BaseRepository {
52
52
  * allowed to re-run selection for a `delivery_pending` cell.
53
53
  */
54
54
  listForCell(deliveryTarget: string, slotId: string, targetId: string): DeliveryRow[];
55
+ /**
56
+ * Every delivery intent recorded for one Slot CELL, across ALL delivery
57
+ * routes. The Slot rollup asks this before it is allowed to converge a cell
58
+ * that was never ACKed; see `SlotCoordinator.finish`.
59
+ */
60
+ listForSlotCell(slotId: string, targetId: string): DeliveryRow[];
55
61
  /**
56
62
  * True when this exact work is already CONFIRMED (delivered, or an attested
57
63
  * historical duplicate) for this target. Pending/failed intents do not block
@@ -55,6 +55,19 @@ class DeliveryRepository extends BaseRepository_1.BaseRepository {
55
55
  .all(deliveryTarget, slotId, targetId);
56
56
  return rows.map((r) => this.toRow(r));
57
57
  }
58
+ /**
59
+ * Every delivery intent recorded for one Slot CELL, across ALL delivery
60
+ * routes. The Slot rollup asks this before it is allowed to converge a cell
61
+ * that was never ACKed; see `SlotCoordinator.finish`.
62
+ */
63
+ listForSlotCell(slotId, targetId) {
64
+ const rows = this.db
65
+ .prepare(`SELECT * FROM deliveries
66
+ WHERE slot_id = ? AND target_id = ?
67
+ ORDER BY created_at ASC`)
68
+ .all(slotId, targetId);
69
+ return rows.map((r) => this.toRow(r));
70
+ }
58
71
  /**
59
72
  * True when this exact work is already CONFIRMED (delivered, or an attested
60
73
  * historical duplicate) for this target. Pending/failed intents do not block
@@ -30,7 +30,41 @@ export interface RecordEventInput {
30
30
  /** Short pre-sanitized JSON-able detail. Never put secrets here. */
31
31
  detail?: Record<string, unknown> | null;
32
32
  }
33
- export type OutboxKind = 'delivery' | 'notification';
33
+ /**
34
+ * A position in one job's durable event log: the `(ts, id)` of a
35
+ * `delivery_events` row. Total order, resolvable back to the row, and stable
36
+ * across restarts — which is what makes the ack cursor meaningful.
37
+ */
38
+ export interface EventCursorPosition {
39
+ at: number;
40
+ rowId: number;
41
+ }
42
+ export interface JobEventCursor extends EventCursorPosition {
43
+ jobId: string;
44
+ updatedAt: number;
45
+ }
46
+ export interface SlotEventInput {
47
+ slotId: string;
48
+ event: string;
49
+ /** Honest instant of the transition. Clamped to the slot's newest event. */
50
+ at?: number;
51
+ /** Only set on the `job.requested` row: the declared callback_url. */
52
+ deliveryTarget?: string | null;
53
+ detail?: Record<string, unknown> | null;
54
+ }
55
+ export interface SlotEventWrite {
56
+ /** False when an identical lifecycle event was already durable. */
57
+ inserted: boolean;
58
+ event: DeliveryEvent;
59
+ }
60
+ /**
61
+ * Outbox row kinds. `event_callback` is a Workflow Protocol v1 §events
62
+ * obligation: deliver one `$defs/Event` to the Task's `callback_url`. It rides
63
+ * the SAME outbox, dedupe index and retry machinery as outcome delivery — it is
64
+ * not a second delivery system, and `hasActionableDelivery` (kind='delivery')
65
+ * and the delivery ledger are deliberately unaffected by it.
66
+ */
67
+ export type OutboxKind = 'delivery' | 'notification' | 'event_callback';
34
68
  export type OutboxStatus = 'pending' | 'processing' | 'retry_wait' | 'done' | 'dead' | 'cancelled';
35
69
  export interface OutboxRow {
36
70
  id: string;
@@ -145,6 +179,44 @@ export declare class OutboxRepository extends BaseRepository {
145
179
  * `DELETE FROM delivery_events WHERE ts < ?` there (same age constant).
146
180
  */
147
181
  recordEvent(input: RecordEventInput, now?: number): void;
182
+ /**
183
+ * Append one job-lifecycle event at most once.
184
+ *
185
+ * Idempotent by construction: a single `INSERT ... SELECT ... WHERE NOT
186
+ * EXISTS` statement, so two callers cannot race a duplicate into the log, and
187
+ * no UNIQUE index is needed (an index over historical duplicate rows would
188
+ * brick the migration instead of reporting them).
189
+ */
190
+ recordSlotEventOnce(input: SlotEventInput, now?: number): SlotEventWrite;
191
+ /** Newest recorded `ts` for a slot; null when the slot has no events yet. */
192
+ newestSlotEventTs(slotId: string): number | null;
193
+ /** Newest row of one internal kind for a slot. */
194
+ slotEvent(slotId: string, event: string): DeliveryEvent | null;
195
+ /** One row by its durable `(slot_id, id)` identity — the ack resolver. */
196
+ slotEventById(slotId: string, id: number): DeliveryEvent | null;
197
+ /** A job's events of the given internal kinds, oldest first (ascending `at`). */
198
+ listSlotEvents(slotId: string, kinds: readonly string[], after?: EventCursorPosition | null, limit?: number): DeliveryEvent[];
199
+ /** How many of a job's events sit strictly after a cursor position. */
200
+ countSlotEvents(slotId: string, kinds: readonly string[], after?: EventCursorPosition | null): number;
201
+ /**
202
+ * Slots that declared a callback_url but whose terminal event is not durable
203
+ * yet. This is the sweep behind "a callback is never silently dropped": it is
204
+ * driven from durable rows only, so a crash between "job terminal" and
205
+ * "callback enqueued" is repaired, and it converges (each pass either writes
206
+ * the terminal event or the job is not terminal yet).
207
+ */
208
+ slotsAwaitingTerminalEvent(terminalKinds: readonly string[], limit?: number): string[];
209
+ /** The consumer's durable ack cursor for a job, or null when never acked. */
210
+ jobEventCursor(jobId: string): JobEventCursor | null;
211
+ /**
212
+ * Advance a job's ack cursor. Monotonic and idempotent: a position that is not
213
+ * strictly ahead of the stored one is a no-op and the stored position is
214
+ * returned unchanged. This only records what the consumer attests it has
215
+ * durably stored — it never touches the slot ledger.
216
+ */
217
+ advanceJobEventCursor(jobId: string, position: EventCursorPosition, now?: number): JobEventCursor;
218
+ private jobEventFilter;
219
+ private toCursor;
148
220
  listEvents(filter?: {
149
221
  executionId?: string;
150
222
  outboxId?: string;
@@ -297,6 +297,148 @@ class OutboxRepository extends BaseRepository_1.BaseRepository {
297
297
  detail: input.detail ? JSON.stringify(input.detail) : null,
298
298
  });
299
299
  }
300
+ // --- job event stream (Workflow Protocol v1 §events) ---
301
+ //
302
+ // The protocol event stream is a PROJECTION of this same durable log: no
303
+ // second event table, no second queue. `slot_id` is the job id, `event` is the
304
+ // internal kind, `ts` is the protocol `at`. Writers below guarantee the
305
+ // once-only and monotonic properties the protocol requires.
306
+ /**
307
+ * Append one job-lifecycle event at most once.
308
+ *
309
+ * Idempotent by construction: a single `INSERT ... SELECT ... WHERE NOT
310
+ * EXISTS` statement, so two callers cannot race a duplicate into the log, and
311
+ * no UNIQUE index is needed (an index over historical duplicate rows would
312
+ * brick the migration instead of reporting them).
313
+ */
314
+ recordSlotEventOnce(input, now = Date.now()) {
315
+ // `at` is never allowed to move backwards: the page is ordered by `at` and
316
+ // `next_after` is a row position, so ascending timestamps and ascending ids
317
+ // must agree.
318
+ const ts = Math.max(input.at ?? now, this.newestSlotEventTs(input.slotId) ?? 0);
319
+ const info = this.db
320
+ .prepare(`INSERT INTO delivery_events
321
+ (ts, slot_id, delivery_target, event, counts_as_attempt, actor, detail)
322
+ SELECT @ts, @slotId, @deliveryTarget, @event, 0, NULL, @detail
323
+ WHERE NOT EXISTS (
324
+ SELECT 1 FROM delivery_events WHERE slot_id = @slotId AND event = @event
325
+ )`)
326
+ .run({
327
+ ts,
328
+ slotId: input.slotId,
329
+ deliveryTarget: input.deliveryTarget ?? null,
330
+ event: input.event,
331
+ detail: input.detail ? JSON.stringify(input.detail) : null,
332
+ });
333
+ const event = this.slotEvent(input.slotId, input.event);
334
+ if (!event)
335
+ throw new Error(`job event ${input.event} for ${input.slotId} was not persisted`);
336
+ return { inserted: info.changes > 0, event };
337
+ }
338
+ /** Newest recorded `ts` for a slot; null when the slot has no events yet. */
339
+ newestSlotEventTs(slotId) {
340
+ const row = this.db
341
+ .prepare('SELECT MAX(ts) AS ts FROM delivery_events WHERE slot_id = ?')
342
+ .get(slotId);
343
+ return row?.ts ?? null;
344
+ }
345
+ /** Newest row of one internal kind for a slot. */
346
+ slotEvent(slotId, event) {
347
+ const row = this.db
348
+ .prepare('SELECT * FROM delivery_events WHERE slot_id = ? AND event = ? ORDER BY ts DESC, id DESC LIMIT 1')
349
+ .get(slotId, event);
350
+ return row ? this.toEvent(row) : null;
351
+ }
352
+ /** One row by its durable `(slot_id, id)` identity — the ack resolver. */
353
+ slotEventById(slotId, id) {
354
+ const row = this.db
355
+ .prepare('SELECT * FROM delivery_events WHERE slot_id = ? AND id = ?')
356
+ .get(slotId, id);
357
+ return row ? this.toEvent(row) : null;
358
+ }
359
+ /** A job's events of the given internal kinds, oldest first (ascending `at`). */
360
+ listSlotEvents(slotId, kinds, after = null, limit = 200) {
361
+ if (kinds.length === 0)
362
+ return [];
363
+ const { where, params } = this.jobEventFilter(slotId, kinds, after);
364
+ const rows = this.db
365
+ .prepare(`SELECT * FROM delivery_events WHERE ${where} ORDER BY ts ASC, id ASC LIMIT ?`)
366
+ .all(...params, Math.max(1, Math.min(limit, 500)));
367
+ return rows.map((r) => this.toEvent(r));
368
+ }
369
+ /** How many of a job's events sit strictly after a cursor position. */
370
+ countSlotEvents(slotId, kinds, after = null) {
371
+ if (kinds.length === 0)
372
+ return 0;
373
+ const { where, params } = this.jobEventFilter(slotId, kinds, after);
374
+ const row = this.db
375
+ .prepare(`SELECT COUNT(*) AS n FROM delivery_events WHERE ${where}`)
376
+ .get(...params);
377
+ return row.n;
378
+ }
379
+ /**
380
+ * Slots that declared a callback_url but whose terminal event is not durable
381
+ * yet. This is the sweep behind "a callback is never silently dropped": it is
382
+ * driven from durable rows only, so a crash between "job terminal" and
383
+ * "callback enqueued" is repaired, and it converges (each pass either writes
384
+ * the terminal event or the job is not terminal yet).
385
+ */
386
+ slotsAwaitingTerminalEvent(terminalKinds, limit = 50) {
387
+ if (terminalKinds.length === 0)
388
+ return [];
389
+ const placeholders = terminalKinds.map(() => '?').join(', ');
390
+ const rows = this.db
391
+ .prepare(`SELECT e.slot_id AS slot_id
392
+ FROM delivery_events e
393
+ WHERE e.event = 'job.requested'
394
+ AND e.delivery_target IS NOT NULL
395
+ AND e.slot_id IS NOT NULL
396
+ AND NOT EXISTS (
397
+ SELECT 1 FROM delivery_events t
398
+ WHERE t.slot_id = e.slot_id AND t.event IN (${placeholders})
399
+ )
400
+ GROUP BY e.slot_id
401
+ ORDER BY MAX(e.ts) DESC
402
+ LIMIT ?`)
403
+ .all(...terminalKinds, Math.max(1, Math.min(limit, 500)));
404
+ return rows.map((r) => r.slot_id);
405
+ }
406
+ /** The consumer's durable ack cursor for a job, or null when never acked. */
407
+ jobEventCursor(jobId) {
408
+ const row = this.db.prepare('SELECT * FROM job_event_cursors WHERE job_id = ?').get(jobId);
409
+ return row ? this.toCursor(row) : null;
410
+ }
411
+ /**
412
+ * Advance a job's ack cursor. Monotonic and idempotent: a position that is not
413
+ * strictly ahead of the stored one is a no-op and the stored position is
414
+ * returned unchanged. This only records what the consumer attests it has
415
+ * durably stored — it never touches the slot ledger.
416
+ */
417
+ advanceJobEventCursor(jobId, position, now = Date.now()) {
418
+ this.db
419
+ .prepare(`INSERT INTO job_event_cursors (job_id, acked_at, acked_row_id, updated_at)
420
+ VALUES (@jobId, @at, @rowId, @now)
421
+ ON CONFLICT(job_id) DO UPDATE SET
422
+ acked_at = @at,
423
+ acked_row_id = @rowId,
424
+ updated_at = @now
425
+ WHERE @at > acked_at OR (@at = acked_at AND @rowId > acked_row_id)`)
426
+ .run({ jobId, at: position.at, rowId: position.rowId, now });
427
+ return this.jobEventCursor(jobId);
428
+ }
429
+ jobEventFilter(slotId, kinds, after) {
430
+ const placeholders = kinds.map(() => '?').join(', ');
431
+ const clauses = ['slot_id = ?', `event IN (${placeholders})`];
432
+ const params = [slotId, ...kinds];
433
+ if (after) {
434
+ clauses.push('(ts > ? OR (ts = ? AND id > ?))');
435
+ params.push(after.at, after.at, after.rowId);
436
+ }
437
+ return { where: clauses.join(' AND '), params };
438
+ }
439
+ toCursor(row) {
440
+ return { jobId: row.job_id, at: row.acked_at, rowId: row.acked_row_id, updatedAt: row.updated_at };
441
+ }
300
442
  listEvents(filter = {}) {
301
443
  const clauses = [];
302
444
  const params = [];