pixivflow 2.20.4 → 2.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/README.en.md +180 -53
  2. package/README.md +113 -59
  3. package/dist/commands/SchedulerCommand.js +66 -0
  4. package/dist/commands/SchedulerIdleLifecycle.d.ts +17 -0
  5. package/dist/commands/SchedulerIdleLifecycle.js +3 -1
  6. package/dist/commands/scheduler-runtime.d.ts +8 -0
  7. package/dist/commands/scheduler-runtime.js +121 -94
  8. package/dist/config/types.d.ts +26 -0
  9. package/dist/config/validation.js +21 -0
  10. package/dist/delivery/types.d.ts +15 -0
  11. package/dist/notification/NotificationPolicy.d.ts +2 -0
  12. package/dist/notification/NotificationPolicy.js +18 -1
  13. package/dist/package.json +1 -1
  14. package/dist/scheduler/MultiScheduleManager.d.ts +12 -1
  15. package/dist/scheduler/MultiScheduleManager.js +46 -50
  16. package/dist/scheduler/RecoveryPolicy.d.ts +45 -0
  17. package/dist/scheduler/RecoveryPolicy.js +63 -0
  18. package/dist/scheduler/ResourceAdmission.d.ts +68 -0
  19. package/dist/scheduler/ResourceAdmission.js +83 -0
  20. package/dist/scheduler/ScheduleTriggerServer.d.ts +16 -0
  21. package/dist/scheduler/ScheduleTriggerServer.js +55 -0
  22. package/dist/scheduler/Scheduler.d.ts +19 -2
  23. package/dist/scheduler/Scheduler.js +12 -3
  24. package/dist/scheduler/SlotCoordinator.d.ts +20 -0
  25. package/dist/scheduler/SlotCoordinator.js +44 -2
  26. package/dist/scheduler/TargetOutcome.d.ts +31 -0
  27. package/dist/scheduler/TargetOutcome.js +114 -0
  28. package/dist/storage/DatabaseMigration.js +17 -0
  29. package/dist/storage/repositories/SlotRepository.d.ts +29 -0
  30. package/dist/storage/repositories/SlotRepository.js +28 -2
  31. package/dist/version.js +1 -1
  32. package/dist/webui/package.json +1 -1
  33. package/package.json +2 -2
@@ -9,6 +9,7 @@ const node_fs_1 = require("node:fs");
9
9
  const cron_parser_1 = __importDefault(require("cron-parser"));
10
10
  const logger_1 = require("../logger");
11
11
  const Scheduler_1 = require("./Scheduler");
12
+ const ResourceAdmission_1 = require("./ResourceAdmission");
12
13
  const schedules_1 = require("./schedules");
13
14
  const OccurrenceResolver_1 = require("./OccurrenceResolver");
14
15
  /**
@@ -23,57 +24,26 @@ function asTriggerSource(value) {
23
24
  ? value
24
25
  : 'catchup';
25
26
  }
26
- class SerialJobAdmission {
27
- queueLimit;
28
- active = false;
29
- pendingIds = new Set();
30
- queue = [];
31
- constructor(queueLimit) {
32
- this.queueLimit = queueLimit;
33
- }
34
- setQueueLimit(queueLimit) {
35
- this.queueLimit = Math.max(0, queueLimit);
36
- }
37
- acquire(scheduleId) {
38
- if (!this.active) {
39
- this.active = true;
40
- return Promise.resolve(this.createLease());
41
- }
42
- // A slow task may span multiple cron ticks. Retain at most one pending run
43
- // for each plan so a temporary outage cannot create an unbounded backlog.
44
- if (this.pendingIds.has(scheduleId) || this.queue.length >= this.queueLimit) {
45
- return Promise.resolve(null);
46
- }
47
- this.pendingIds.add(scheduleId);
48
- return new Promise((resolve) => {
49
- this.queue.push({ scheduleId, resolve });
50
- });
51
- }
52
- createLease() {
53
- let released = false;
54
- return {
55
- release: () => {
56
- if (released)
57
- return;
58
- released = true;
59
- this.releaseNext();
60
- },
61
- };
62
- }
63
- releaseNext() {
64
- const next = this.queue.shift();
65
- if (!next) {
66
- this.active = false;
67
- return;
68
- }
69
- this.pendingIds.delete(next.scheduleId);
70
- next.resolve(this.createLease());
71
- }
27
+ /**
28
+ * Capacity resolver for one resource key, from the configuration's
29
+ * resource-governance block (§resource-governance). Unknown resources default
30
+ * to capacity 1. The key encodes the resource identity (`pixiv-account:<id>`),
31
+ * so two different accounts get two independent capacities.
32
+ */
33
+ function capacityOfResourceKey(config, key) {
34
+ const prefix = 'pixiv-account:';
35
+ if (!key.startsWith(prefix))
36
+ return 1;
37
+ const accountId = key.slice(prefix.length);
38
+ const profile = config?.schedulerRuntime?.resourceGovernance?.pixivAccounts?.[accountId];
39
+ return Math.max(1, profile?.maxConcurrency ?? 1);
72
40
  }
73
41
  /**
74
42
  * Hosts many cron plans in one process. A validated config snapshot replaces
75
43
  * the complete cron table at once; invalid updates leave the previous table
76
- * running. All jobs share one bounded serial admission queue by default.
44
+ * running. All jobs that consume the same constrained resource (today: the
45
+ * shared Pixiv account) pass through ONE bounded resource admission; work for
46
+ * different resource identities runs concurrently.
77
47
  */
78
48
  class MultiScheduleManager {
79
49
  options;
@@ -83,9 +53,25 @@ class MultiScheduleManager {
83
53
  reloadTimer = null;
84
54
  recoveryTimer = null;
85
55
  watching = false;
86
- admission = new SerialJobAdmission(8);
56
+ admission;
87
57
  constructor(options) {
88
58
  this.options = options;
59
+ // Capacity is read lazily from the current config snapshot so a hot
60
+ // reload that changes `resourceGovernance` applies immediately.
61
+ this.admission = new ResourceAdmission_1.ResourceAdmission((key) => capacityOfResourceKey(this.activeConfig, key), 8);
62
+ }
63
+ /**
64
+ * Deduplicated, log-safe observable: how many work items are parked waiting
65
+ * for any resource. Used by the idle lifecycle — waiting work is unfinished
66
+ * work and must prevent premature shutdown (§waiting-is-not-idle).
67
+ */
68
+ waitingWorkCount() {
69
+ return this.admission.waitingTotal();
70
+ }
71
+ /** The resource key a plan's work consumes (resource identity, never a bot/
72
+ * schedule/target name). All Pixiv-consuming work shares the account id. */
73
+ resourceKeyForPlan() {
74
+ return (0, ResourceAdmission_1.pixivAccountResourceKey)(this.activeConfig?.pixiv?.accountId);
89
75
  }
90
76
  start(initialConfig) {
91
77
  const config = initialConfig ?? this.options.loadConfig();
@@ -180,6 +166,12 @@ class MultiScheduleManager {
180
166
  slotDate: slot.slotDate || slot.occurrenceDate,
181
167
  manualRequestId: slot.manualRequestId ?? undefined,
182
168
  correlationId: slot.correlationId ?? undefined,
169
+ // A crashed manual recovery resumes with the SAME occurrence-scoped
170
+ // policy, never a re-resolution that would lose the override.
171
+ recoveryRequestId: slot.recoveryRequestId ?? undefined,
172
+ recoveryMode: slot.recoveryMode === 'relaxed' || slot.recoveryMode === 'normal'
173
+ ? slot.recoveryMode
174
+ : undefined,
183
175
  };
184
176
  const admitted = this.triggerSchedule(slot.scheduleId, {
185
177
  triggerSource: context.triggerSource,
@@ -284,12 +276,15 @@ class MultiScheduleManager {
284
276
  this.activeConfig = config;
285
277
  this.generation++;
286
278
  const generation = this.generation;
287
- this.admission.setQueueLimit(queueLimit);
279
+ // Bounded wait queue across ALL resources (mirrors the legacy queueLimit
280
+ // guard against an unbounded backlog); capacity itself is per resource key.
281
+ this.admission.setMaxWaiting(queueLimit);
282
+ const resourceKey = this.resourceKeyForPlan();
288
283
  for (const plan of enabledPlans) {
289
284
  // Watchdog: schedules without an explicit timeout still get a cap so a
290
285
  // wedged run cannot hold the shared admission queue forever.
291
286
  const schedulerConfig = plan.timeout !== undefined ? plan : { ...plan, timeout: Scheduler_1.DEFAULT_SCHEDULE_TIMEOUT_MS };
292
- const scheduler = new Scheduler_1.Scheduler(schedulerConfig, this.options.database, this.options.telemetry, plan.id, this.admission, (failure) => this.options.onFailure?.(config, plan, failure), (abandoned) => this.options.onAbandoned?.(config, plan, abandoned));
287
+ const scheduler = new Scheduler_1.Scheduler(schedulerConfig, this.options.database, this.options.telemetry, plan.id, this.admission, (failure) => this.options.onFailure?.(config, plan, failure), (abandoned) => this.options.onAbandoned?.(config, plan, abandoned), resourceKey);
293
288
  scheduler[registerCron ? 'start' : 'init'](async (options) => {
294
289
  // The closure keeps the exact validated snapshot for an in-flight run.
295
290
  // Cron fires pass no options (runJob resolves the occurrence for now);
@@ -308,6 +303,7 @@ class MultiScheduleManager {
308
303
  targets: plan.targetIds?.length ?? 'all',
309
304
  })),
310
305
  queueLimit,
306
+ resourceKey,
311
307
  });
312
308
  return { ok: true, generation, schedules: scheduleIds };
313
309
  }
@@ -0,0 +1,45 @@
1
+ /**
2
+ * Execution / acquisition policy preset (§recovery-policy).
3
+ *
4
+ * Operators do not tune `lookbackDays` / `candidateScanLimit` / language
5
+ * windows per attempt. The server defines a small number of named presets —
6
+ * `normal` and `relaxed` — and a recovery run reads the "effective policy"
7
+ * derived from the base config plus the occurrence-scoped override.
8
+ *
9
+ * Hard business constraints are NEVER relaxed by an ordinary recovery policy:
10
+ * security rules, explicit bans, wrong media type, data-integrity requirements,
11
+ * permission constraints, the selected topic/work-type boundary, delivery
12
+ * wiring and already-successfully-submitted exact duplicates are untouched.
13
+ * Only SOFT criteria (search range, candidate scan count, language-candidate
14
+ * window) may widen.
15
+ *
16
+ * The override is occurrence-scoped: `applyAcquisitionPolicy` is a pure
17
+ * function over a target snapshot. Nothing here ever writes the global config,
18
+ * so a manual relaxed recovery can never change future scheduled runs.
19
+ */
20
+ import { TargetConfig } from '../config';
21
+ export type RecoveryMode = 'normal' | 'relaxed';
22
+ /** The soft levers an acquisition policy may adjust. */
23
+ export interface AcquisitionPolicy {
24
+ /** Multiplier over the effective candidate scan limit (capped at 100). */
25
+ scanLimitMultiplier: number;
26
+ /** Multiplier over `noMatchPolicy.lookbackDays` (capped at the config max). */
27
+ lookbackDaysMultiplier: number;
28
+ /** Multiplier over the novel `languageCandidateLimit` search window. */
29
+ languageCandidateLimitMultiplier: number;
30
+ }
31
+ /**
32
+ * Server-defined presets. `normal` is the identity policy (the base config);
33
+ * `relaxed` widens only the soft criteria above.
34
+ */
35
+ export declare const ACQUISITION_POLICIES: Record<RecoveryMode, AcquisitionPolicy>;
36
+ /** Cap on the relaxed scan limit, mirroring the schedule fallback cap. */
37
+ export declare const RELAXED_SCAN_LIMIT_CAP = 100;
38
+ /** Cap on lookback days widened by a policy (matches the config validation cap). */
39
+ export declare const RELAXED_LOOKBACK_DAYS_CAP = 7;
40
+ /**
41
+ * Apply an acquisition policy to ONE target snapshot (pure; never mutates the
42
+ * input or global config). `normal` returns the target unchanged.
43
+ */
44
+ export declare function applyAcquisitionPolicy(target: TargetConfig, mode: RecoveryMode): TargetConfig;
45
+ //# sourceMappingURL=RecoveryPolicy.d.ts.map
@@ -0,0 +1,63 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.RELAXED_LOOKBACK_DAYS_CAP = exports.RELAXED_SCAN_LIMIT_CAP = exports.ACQUISITION_POLICIES = void 0;
4
+ exports.applyAcquisitionPolicy = applyAcquisitionPolicy;
5
+ /**
6
+ * Server-defined presets. `normal` is the identity policy (the base config);
7
+ * `relaxed` widens only the soft criteria above.
8
+ */
9
+ exports.ACQUISITION_POLICIES = {
10
+ normal: {
11
+ scanLimitMultiplier: 1,
12
+ lookbackDaysMultiplier: 1,
13
+ languageCandidateLimitMultiplier: 1,
14
+ },
15
+ relaxed: {
16
+ scanLimitMultiplier: 3,
17
+ lookbackDaysMultiplier: 3,
18
+ languageCandidateLimitMultiplier: 2,
19
+ },
20
+ };
21
+ /** Cap on the relaxed scan limit, mirroring the schedule fallback cap. */
22
+ exports.RELAXED_SCAN_LIMIT_CAP = 100;
23
+ /** Cap on lookback days widened by a policy (matches the config validation cap). */
24
+ exports.RELAXED_LOOKBACK_DAYS_CAP = 7;
25
+ /**
26
+ * Apply an acquisition policy to ONE target snapshot (pure; never mutates the
27
+ * input or global config). `normal` returns the target unchanged.
28
+ */
29
+ function applyAcquisitionPolicy(target, mode) {
30
+ if (mode === 'normal')
31
+ return target;
32
+ const policy = exports.ACQUISITION_POLICIES[mode];
33
+ const scanLimit = target.candidateScanLimit !== undefined
34
+ ? cap(MULTIPLY(target.candidateScanLimit, policy.scanLimitMultiplier), exports.RELAXED_SCAN_LIMIT_CAP)
35
+ : undefined;
36
+ const languageCandidateLimit = target.languageCandidateLimit !== undefined
37
+ ? MULTIPLY(target.languageCandidateLimit, policy.languageCandidateLimitMultiplier)
38
+ : undefined;
39
+ const lookbackDays = widenLookback(target.noMatchPolicy?.lookbackDays, policy.lookbackDaysMultiplier);
40
+ if (scanLimit === undefined && languageCandidateLimit === undefined && lookbackDays === undefined) {
41
+ return target;
42
+ }
43
+ return {
44
+ ...target,
45
+ ...(scanLimit !== undefined ? { candidateScanLimit: scanLimit } : {}),
46
+ ...(languageCandidateLimit !== undefined ? { languageCandidateLimit } : {}),
47
+ ...(lookbackDays !== undefined
48
+ ? { noMatchPolicy: { ...(target.noMatchPolicy ?? {}), lookbackDays } }
49
+ : {}),
50
+ };
51
+ }
52
+ function MULTIPLY(value, multiplier) {
53
+ return Math.max(1, Math.floor(value * multiplier));
54
+ }
55
+ function cap(value, maximum) {
56
+ return Math.min(value, maximum);
57
+ }
58
+ function widenLookback(value, multiplier) {
59
+ if (value === undefined)
60
+ return undefined;
61
+ return Math.min(MULTIPLY(value, multiplier), exports.RELAXED_LOOKBACK_DAYS_CAP);
62
+ }
63
+ //# sourceMappingURL=RecoveryPolicy.js.map
@@ -0,0 +1,68 @@
1
+ /**
2
+ * Resource-scoped execution admission (§resource-governance).
3
+ *
4
+ * Concurrency is governed by the constrained RESOURCE a work item consumes
5
+ * (today: a Pixiv account), never by bot, schedule or target. Every work item
6
+ * that touches the scarce upstream — scheduled acquisition, fallback passes,
7
+ * manual refetch, manual normal/relaxed recovery — declares the resource key
8
+ * it needs and passes through this same bounded-capacity admission.
9
+ *
10
+ * Design constraints honoured here:
11
+ *
12
+ * - Resource identity is a stable internal profile key (e.g.
13
+ * `pixiv-account:<accountId>`). Tokens/cookies/session values are never a
14
+ * resource key and never appear in logs.
15
+ * - Capacity is config-driven per resource identity; the scheduler code does
16
+ * not hardcode `1`.
17
+ * - Waiting is FIFO (arrival order) per resource key, so no producer is
18
+ * starved. Capacity is checked at the moment a lease is handed out, so a
19
+ * single process never exceeds `capacity` active leases for one key.
20
+ * - Different resource identities never block each other: account A and
21
+ * account B run concurrently under independent capacities (no global lock).
22
+ * - Waiting is NOT a failure: `acquire` resolves with `null` only when the
23
+ * bounded wait queue itself is full (the caller then leaves the work in its
24
+ * durable ledger for the recovery sweep). A queued work item is unfinished
25
+ * work — it must keep the idle-detector from shutting the worker down
26
+ * (see `waitingTotal` on the idle snapshot).
27
+ *
28
+ * Implementation level matches the production topology: one PixivFlow process
29
+ * owns the whole ledger on one machine, so an in-process FIFO semaphore is the
30
+ * correct mechanism. A durable/distributed lease would only be needed if the
31
+ * same resource could be consumed by multiple processes at once.
32
+ */
33
+ export interface ResourceLease {
34
+ release(): void;
35
+ }
36
+ /**
37
+ * FIFO per-resource admission with config-driven capacity.
38
+ *
39
+ * `capacityOf(resourceKey)` returns the configured capacity for that key
40
+ * (default 1). `maxWaiting` bounds the TOTAL number of parked work items
41
+ * across all resources, mirroring the legacy `schedulerRuntime.queueLimit`
42
+ * guard against an unbounded backlog.
43
+ */
44
+ export declare class ResourceAdmission {
45
+ private readonly capacityOf;
46
+ private maxWaiting;
47
+ private readonly active;
48
+ private readonly waiting;
49
+ constructor(capacityOf: (resourceKey: string) => number, maxWaiting?: number);
50
+ setMaxWaiting(maxWaiting: number): void;
51
+ /**
52
+ * Acquire a lease on `resourceKey`. Resolves immediately when capacity is
53
+ * available, parks the caller in a FIFO per-key queue otherwise, and resolves
54
+ * `null` (never rejects) when the bounded wait queue is full.
55
+ */
56
+ acquire(resourceKey: string): Promise<ResourceLease | null>;
57
+ private createLease;
58
+ private releaseNext;
59
+ /** Active leases for one resource key. */
60
+ activeCount(resourceKey: string): number;
61
+ /** Work items parked waiting for one resource key. */
62
+ waitingCount(resourceKey: string): number;
63
+ /** Work items parked waiting for ANY resource. Unfinished work, not failure. */
64
+ waitingTotal(): number;
65
+ }
66
+ /** The canonical resource identity for the process's Pixiv credential profile. */
67
+ export declare function pixivAccountResourceKey(accountId: string | undefined): string;
68
+ //# sourceMappingURL=ResourceAdmission.d.ts.map
@@ -0,0 +1,83 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.ResourceAdmission = void 0;
4
+ exports.pixivAccountResourceKey = pixivAccountResourceKey;
5
+ /**
6
+ * FIFO per-resource admission with config-driven capacity.
7
+ *
8
+ * `capacityOf(resourceKey)` returns the configured capacity for that key
9
+ * (default 1). `maxWaiting` bounds the TOTAL number of parked work items
10
+ * across all resources, mirroring the legacy `schedulerRuntime.queueLimit`
11
+ * guard against an unbounded backlog.
12
+ */
13
+ class ResourceAdmission {
14
+ capacityOf;
15
+ maxWaiting;
16
+ active = new Map();
17
+ waiting = [];
18
+ constructor(capacityOf, maxWaiting = 8) {
19
+ this.capacityOf = capacityOf;
20
+ this.maxWaiting = maxWaiting;
21
+ }
22
+ setMaxWaiting(maxWaiting) {
23
+ this.maxWaiting = Math.max(0, maxWaiting);
24
+ }
25
+ /**
26
+ * Acquire a lease on `resourceKey`. Resolves immediately when capacity is
27
+ * available, parks the caller in a FIFO per-key queue otherwise, and resolves
28
+ * `null` (never rejects) when the bounded wait queue is full.
29
+ */
30
+ acquire(resourceKey) {
31
+ if (this.activeCount(resourceKey) < this.capacityOf(resourceKey)) {
32
+ this.active.set(resourceKey, this.activeCount(resourceKey) + 1);
33
+ return Promise.resolve(this.createLease(resourceKey));
34
+ }
35
+ if (this.waiting.length >= this.maxWaiting) {
36
+ return Promise.resolve(null);
37
+ }
38
+ return new Promise((resolve) => {
39
+ this.waiting.push({ resourceKey, resolve });
40
+ });
41
+ }
42
+ createLease(resourceKey) {
43
+ let released = false;
44
+ return {
45
+ release: () => {
46
+ if (released)
47
+ return;
48
+ released = true;
49
+ this.releaseNext(resourceKey);
50
+ },
51
+ };
52
+ }
53
+ releaseNext(resourceKey) {
54
+ // Hand the freed slot to the OLDEST waiter for this resource key (FIFO),
55
+ // skipping waiters of other resources so they never block this one.
56
+ const index = this.waiting.findIndex((entry) => entry.resourceKey === resourceKey);
57
+ if (index === -1) {
58
+ const remaining = Math.max(0, this.activeCount(resourceKey) - 1);
59
+ this.active.set(resourceKey, remaining);
60
+ return;
61
+ }
62
+ const [entry] = this.waiting.splice(index, 1);
63
+ entry.resolve(this.createLease(resourceKey));
64
+ }
65
+ /** Active leases for one resource key. */
66
+ activeCount(resourceKey) {
67
+ return this.active.get(resourceKey) ?? 0;
68
+ }
69
+ /** Work items parked waiting for one resource key. */
70
+ waitingCount(resourceKey) {
71
+ return this.waiting.filter((entry) => entry.resourceKey === resourceKey).length;
72
+ }
73
+ /** Work items parked waiting for ANY resource. Unfinished work, not failure. */
74
+ waitingTotal() {
75
+ return this.waiting.length;
76
+ }
77
+ }
78
+ exports.ResourceAdmission = ResourceAdmission;
79
+ /** The canonical resource identity for the process's Pixiv credential profile. */
80
+ function pixivAccountResourceKey(accountId) {
81
+ return `pixiv-account:${(accountId ?? 'default').trim() || 'default'}`;
82
+ }
83
+ //# sourceMappingURL=ResourceAdmission.js.map
@@ -111,6 +111,22 @@ export interface TriggerHandlers {
111
111
  state: string;
112
112
  slotStatus: string;
113
113
  } | null;
114
+ /**
115
+ * Admit one manual RECOVERY of a failed target (§manual-recovery). `retryMode`
116
+ * selects a SERVER-DEFINED acquisition policy preset ('normal' | 'relaxed');
117
+ * callers never supply raw acquisition parameters. The run is
118
+ * occurrence-scoped: it never changes the global config or future schedules.
119
+ */
120
+ recover?(targetId: string, requestId: string, retryMode: 'normal' | 'relaxed', correlationId?: string): Promise<{
121
+ slotId: string;
122
+ disposition: string;
123
+ }>;
124
+ recoverStatus?(targetId: string, requestId: string): {
125
+ requestId: string;
126
+ slotId: string;
127
+ state: string;
128
+ slotStatus: string;
129
+ } | null;
114
130
  }
115
131
  export declare class ScheduleTriggerServer {
116
132
  private readonly token;
@@ -207,6 +207,61 @@ class ScheduleTriggerServer {
207
207
  }
208
208
  res.json(status);
209
209
  });
210
+ // Manual RECOVERY of a failed schedule target (§manual-recovery). Same
211
+ // authenticated manual-work family as refetch (PIXIVFLOW_REFETCH_TOKEN) but
212
+ // a DIFFERENT business intent: it re-runs the failed target(s) under a
213
+ // server-defined acquisition policy preset and reports through the
214
+ // schedule-outcome channel, so the daily summary is never rewritten.
215
+ // Admitting is all this endpoint does; if the resource is busy the run is
216
+ // queued (202 + disposition 'queued'), never failed.
217
+ app.post('/internal/targets/:targetId/recover', this.refetchAuth, async (req, res) => {
218
+ const requestId = req.body?.requestId;
219
+ if (typeof requestId !== 'string' || !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(requestId)) {
220
+ res.status(400).json({ status: 'error', error: 'requestId must be a UUID' });
221
+ return;
222
+ }
223
+ // Only a NAMED preset is accepted — never raw acquisition parameters.
224
+ const rawMode = req.body?.retryMode;
225
+ const retryMode = rawMode === undefined || rawMode === null || rawMode === '' ? 'normal' : rawMode;
226
+ if (retryMode !== 'normal' && retryMode !== 'relaxed') {
227
+ res.status(400).json({ status: 'error', error: 'retryMode must be "normal" or "relaxed"' });
228
+ return;
229
+ }
230
+ const correlationId = req.body?.correlationId;
231
+ if (correlationId !== undefined && (typeof correlationId !== 'string' || correlationId.length > 200)) {
232
+ res.status(400).json({ status: 'error', error: 'correlationId must be a string of at most 200 chars' });
233
+ return;
234
+ }
235
+ if (!this.handlers.recover) {
236
+ res.status(503).json({ status: 'error', error: 'manual recovery is unavailable' });
237
+ return;
238
+ }
239
+ try {
240
+ const result = await this.handlers.recover(req.params.targetId, requestId, retryMode, correlationId ?? undefined);
241
+ res.status(202).json({ status: 'accepted', retryMode, ...result });
242
+ }
243
+ catch (error) {
244
+ const message = error instanceof Error ? error.message : String(error);
245
+ const status = message === 'unknown target' ? 404 : message === 'ambiguous target' ? 409 : 500;
246
+ logger_1.logger.warn('Manual recovery rejected', {
247
+ targetId: req.params.targetId, requestId, retryMode, status, error: message,
248
+ });
249
+ res.status(status).json({ status: 'error', error: status === 500 ? 'recovery admission failed' : message });
250
+ }
251
+ });
252
+ app.get('/internal/targets/:targetId/recover/:requestId', this.refetchAuth, (req, res) => {
253
+ const { targetId, requestId } = req.params;
254
+ if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(requestId)) {
255
+ res.status(400).json({ status: 'error', error: 'requestId must be a UUID' });
256
+ return;
257
+ }
258
+ const status = this.handlers.recoverStatus?.(targetId, requestId);
259
+ if (!status) {
260
+ res.status(404).json({ status: 'error', error: 'manual recovery not found' });
261
+ return;
262
+ }
263
+ res.json(status);
264
+ });
210
265
  // Convergence endpoint: after a machine stop/start, an operator or an
211
266
  // external watcher can ask the process to flush due deliveries/notifications
212
267
  // without running candidate selection. Deployment-agnostic (no platform refs).
@@ -56,7 +56,12 @@ export interface JobAbandoned {
56
56
  }
57
57
  /** Admission is acquired before timeout/accounting starts. */
58
58
  export interface JobAdmissionController {
59
- acquire(scheduleId: string): Promise<JobLease | null>;
59
+ /**
60
+ * Acquire a lease on the work item's constrained RESOURCE. The key is a
61
+ * resource identity (`pixiv-account:<id>`), never a bot/schedule/target
62
+ * name. Resolves `null` only when the bounded wait queue is full.
63
+ */
64
+ acquire(resourceKey: string): Promise<JobLease | null>;
60
65
  }
61
66
  export declare class Scheduler {
62
67
  private readonly config;
@@ -66,6 +71,12 @@ export declare class Scheduler {
66
71
  private readonly admission?;
67
72
  private readonly onFailure?;
68
73
  private readonly onAbandoned?;
74
+ /**
75
+ * The constrained resource this schedule's work consumes (§resource-
76
+ * governance). All schedules whose targets share the same Pixiv account
77
+ * share the same key, so their runs queue under ONE capacity.
78
+ */
79
+ private readonly resourceKey?;
69
80
  private task;
70
81
  private job;
71
82
  private running;
@@ -76,7 +87,13 @@ export declare class Scheduler {
76
87
  private drainHandle;
77
88
  private stopped;
78
89
  private pending;
79
- constructor(config: SchedulerConfig, database?: Database | undefined, telemetry?: JobTelemetry | undefined, scheduleId?: string, admission?: JobAdmissionController | undefined, onFailure?: ((failure: JobFailure) => Promise<void> | void) | undefined, onAbandoned?: ((abandoned: JobAbandoned) => Promise<void> | void) | undefined);
90
+ constructor(config: SchedulerConfig, database?: Database | undefined, telemetry?: JobTelemetry | undefined, scheduleId?: string, admission?: JobAdmissionController | undefined, onFailure?: ((failure: JobFailure) => Promise<void> | void) | undefined, onAbandoned?: ((abandoned: JobAbandoned) => Promise<void> | void) | undefined,
91
+ /**
92
+ * The constrained resource this schedule's work consumes (§resource-
93
+ * governance). All schedules whose targets share the same Pixiv account
94
+ * share the same key, so their runs queue under ONE capacity.
95
+ */
96
+ resourceKey?: string | undefined);
80
97
  start(job: (options?: ScheduleRunOptions) => Promise<void>): void;
81
98
  /**
82
99
  * Arm the scheduler for external/manual triggers (runNow) WITHOUT registering
@@ -33,6 +33,7 @@ class Scheduler {
33
33
  admission;
34
34
  onFailure;
35
35
  onAbandoned;
36
+ resourceKey;
36
37
  task = null;
37
38
  job = null;
38
39
  running = false;
@@ -43,7 +44,13 @@ class Scheduler {
43
44
  drainHandle = null;
44
45
  stopped = false;
45
46
  pending = false;
46
- constructor(config, database, telemetry, scheduleId = 'default', admission, onFailure, onAbandoned) {
47
+ constructor(config, database, telemetry, scheduleId = 'default', admission, onFailure, onAbandoned,
48
+ /**
49
+ * The constrained resource this schedule's work consumes (§resource-
50
+ * governance). All schedules whose targets share the same Pixiv account
51
+ * share the same key, so their runs queue under ONE capacity.
52
+ */
53
+ resourceKey) {
47
54
  this.config = config;
48
55
  this.database = database;
49
56
  this.telemetry = telemetry;
@@ -51,6 +58,7 @@ class Scheduler {
51
58
  this.admission = admission;
52
59
  this.onFailure = onFailure;
53
60
  this.onAbandoned = onAbandoned;
61
+ this.resourceKey = resourceKey;
54
62
  }
55
63
  start(job) {
56
64
  this.arm(job, true);
@@ -149,15 +157,16 @@ class Scheduler {
149
157
  await new Promise((resolve) => setTimeout(resolve, this.config.failureRetryDelay));
150
158
  }
151
159
  this.pending = true;
152
- const lease = this.admission ? await this.admission.acquire(this.scheduleId) : null;
160
+ const lease = this.admission ? await this.admission.acquire(this.resourceKey ?? this.scheduleId) : null;
153
161
  this.pending = false;
154
162
  if (this.stopped) {
155
163
  lease?.release();
156
164
  return;
157
165
  }
158
166
  if (this.admission && !lease) {
159
- logger_1.logger.warn('Skipping scheduled job because the shared scheduler queue is full', {
167
+ logger_1.logger.warn('Work not admitted: the resource wait queue is full; the durable ledger will retry it', {
160
168
  scheduleId: this.scheduleId,
169
+ resourceKey: this.resourceKey,
161
170
  });
162
171
  return;
163
172
  }
@@ -46,12 +46,26 @@ export interface SlotContext {
46
46
  manualRequestId?: string;
47
47
  /** Opaque caller correlation (review chain / review id). Null unless manual. */
48
48
  correlationId?: string;
49
+ /**
50
+ * Remote manual RECOVERY request UUID (§manual-recovery). Present only on
51
+ * slots opened by the authenticated recover endpoint; null for scheduled
52
+ * occurrences and review refetches. A recovery slot re-runs the failed
53
+ * target(s) under an occurrence-scoped recovery policy and reports through
54
+ * the schedule-outcome channel.
55
+ */
56
+ recoveryRequestId?: string;
57
+ /** Recovery policy preset ('normal' | 'relaxed'); null for non-recovery slots. */
58
+ recoveryMode?: 'normal' | 'relaxed';
49
59
  }
50
60
  export interface SlotCellSummary {
51
61
  targetId: string;
52
62
  status: CellStatus;
53
63
  workId: string | null;
54
64
  error?: string | null;
65
+ /** Normalized terminal failure code (§terminal-reason); null when not failed. */
66
+ terminalReasonCode?: string | null;
67
+ /** User-facing business reason message (§terminal-reason). */
68
+ terminalReasonMessage?: string | null;
55
69
  }
56
70
  export interface SlotRunSummary {
57
71
  scheduleId: string;
@@ -233,6 +247,12 @@ export declare class SlotCoordinator {
233
247
  * sole path to the submitted cell state.
234
248
  */
235
249
  applyOutcome(slotId: string, targetId: string, outcome: TargetOutcome): void;
250
+ /**
251
+ * Persist the normalized terminal reason (§terminal-reason) for a cell that
252
+ * just reached a terminal state. Never unwinds the run and never leaks raw
253
+ * error internals: only the stable code + business message are stored.
254
+ */
255
+ private persistTerminalReason;
236
256
  /** Promote a delivery_pending cell to submitted from a confirmed ACK. */
237
257
  markDelivered(slotId: string, targetId: string, workId: string, workType: string): void;
238
258
  /**