pixivflow 2.20.4 → 2.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +180 -53
- package/README.md +113 -59
- package/dist/commands/SchedulerCommand.js +66 -0
- package/dist/commands/SchedulerIdleLifecycle.d.ts +17 -0
- package/dist/commands/SchedulerIdleLifecycle.js +3 -1
- package/dist/commands/scheduler-runtime.d.ts +8 -0
- package/dist/commands/scheduler-runtime.js +121 -94
- package/dist/config/types.d.ts +26 -0
- package/dist/config/validation.js +21 -0
- package/dist/delivery/types.d.ts +15 -0
- package/dist/notification/NotificationPolicy.d.ts +2 -0
- package/dist/notification/NotificationPolicy.js +18 -1
- package/dist/package.json +1 -1
- package/dist/scheduler/MultiScheduleManager.d.ts +12 -1
- package/dist/scheduler/MultiScheduleManager.js +46 -50
- package/dist/scheduler/RecoveryPolicy.d.ts +45 -0
- package/dist/scheduler/RecoveryPolicy.js +63 -0
- package/dist/scheduler/ResourceAdmission.d.ts +68 -0
- package/dist/scheduler/ResourceAdmission.js +83 -0
- package/dist/scheduler/ScheduleTriggerServer.d.ts +16 -0
- package/dist/scheduler/ScheduleTriggerServer.js +55 -0
- package/dist/scheduler/Scheduler.d.ts +19 -2
- package/dist/scheduler/Scheduler.js +12 -3
- package/dist/scheduler/SlotCoordinator.d.ts +20 -0
- package/dist/scheduler/SlotCoordinator.js +44 -2
- package/dist/scheduler/TargetOutcome.d.ts +31 -0
- package/dist/scheduler/TargetOutcome.js +114 -0
- package/dist/storage/DatabaseMigration.js +17 -0
- package/dist/storage/repositories/SlotRepository.d.ts +29 -0
- package/dist/storage/repositories/SlotRepository.js +28 -2
- package/dist/version.js +1 -1
- package/dist/webui/package.json +1 -1
- package/package.json +2 -2
|
@@ -208,6 +208,65 @@ class SchedulerCommand extends Command_1.BaseCommand {
|
|
|
208
208
|
? { requestId, slotId: slot.id, state: cell.status, slotStatus: slot.status }
|
|
209
209
|
: null;
|
|
210
210
|
},
|
|
211
|
+
/**
|
|
212
|
+
* Manual recovery of a FAILED target (§manual-recovery). Distinct
|
|
213
|
+
* from refetch: there is no review chain to replace and no
|
|
214
|
+
* replacement payload — the target is simply re-acquired under a
|
|
215
|
+
* server-defined policy preset, and its outcome is reported through
|
|
216
|
+
* the ordinary schedule-outcome channel so the automatic history is
|
|
217
|
+
* never rewritten.
|
|
218
|
+
*/
|
|
219
|
+
recover: async (targetId, requestId, retryMode, correlationId) => {
|
|
220
|
+
const cfg = resolveConfig();
|
|
221
|
+
const plans = (cfg.schedules ?? []).filter((plan) => plan.enabled !== false && (0, schedules_1.selectScheduleTargets)(cfg.targets, plan).some((target) => target.id === targetId));
|
|
222
|
+
if (plans.length === 0)
|
|
223
|
+
throw new Error('unknown target');
|
|
224
|
+
if (plans.length !== 1)
|
|
225
|
+
throw new Error('ambiguous target');
|
|
226
|
+
const plan = plans[0];
|
|
227
|
+
const target = (0, schedules_1.selectScheduleTargets)(cfg.targets, plan).find((item) => item.id === targetId);
|
|
228
|
+
const deliveryName = target.delivery?.target;
|
|
229
|
+
const delivery = deliveryName ? cfg.delivery?.targets?.[deliveryName] : undefined;
|
|
230
|
+
if (delivery?.type !== 'httpMultipart' || !delivery.scheduleOutcomeUrl?.trim()) {
|
|
231
|
+
throw new Error('schedule outcome endpoint not configured');
|
|
232
|
+
}
|
|
233
|
+
const now = new Date();
|
|
234
|
+
const date = new Intl.DateTimeFormat('en-CA', {
|
|
235
|
+
timeZone: plan.timezone ?? 'UTC', year: 'numeric', month: '2-digit', day: '2-digit',
|
|
236
|
+
}).format(now);
|
|
237
|
+
const slot = {
|
|
238
|
+
slotId: `${plan.id}@recover-${requestId.toLowerCase()}`,
|
|
239
|
+
scheduleId: plan.id,
|
|
240
|
+
occurrenceAt: now.getTime(),
|
|
241
|
+
occurrenceDate: date,
|
|
242
|
+
occurrenceLabel: 'manual',
|
|
243
|
+
timezone: plan.timezone ?? 'UTC',
|
|
244
|
+
triggerSource: 'manual',
|
|
245
|
+
slotName: retryMode === 'relaxed' ? '手动恢复(放宽条件重试)' : '手动恢复(再试一次)',
|
|
246
|
+
slotDate: date,
|
|
247
|
+
recoveryRequestId: requestId,
|
|
248
|
+
recoveryMode: retryMode,
|
|
249
|
+
correlationId: correlationId || undefined,
|
|
250
|
+
};
|
|
251
|
+
const existing = runtime.database.slots.getSlot(slot.slotId);
|
|
252
|
+
if (existing && (existing.scheduleId !== plan.id || existing.targetIds.length !== 1 || existing.targetIds[0] !== targetId)) {
|
|
253
|
+
throw new Error('ambiguous target');
|
|
254
|
+
}
|
|
255
|
+
const prepared = coordinator.prepare(slot, plan, [target]);
|
|
256
|
+
if (prepared.alreadyCompleted)
|
|
257
|
+
return { slotId: slot.slotId, disposition: 'already_completed' };
|
|
258
|
+
// Same resource admission as every other Pixiv-consuming work
|
|
259
|
+
// item: a busy account queues this run, it does not fail it.
|
|
260
|
+
const started = manager.triggerSchedule(plan.id, { slot, onlyTarget: targetId, triggerSource: 'manual' });
|
|
261
|
+
return { slotId: slot.slotId, disposition: started ? 'accepted' : 'queued' };
|
|
262
|
+
},
|
|
263
|
+
recoverStatus: (targetId, requestId) => {
|
|
264
|
+
const slot = runtime.database.slots.findRecoverySlot(requestId, targetId);
|
|
265
|
+
const cell = slot && runtime.database.slots.getCell(slot.id, targetId);
|
|
266
|
+
return slot && cell
|
|
267
|
+
? { requestId, slotId: slot.id, state: cell.status, slotStatus: slot.status }
|
|
268
|
+
: null;
|
|
269
|
+
},
|
|
211
270
|
status: (scheduleId) => {
|
|
212
271
|
const cfg = resolveConfig();
|
|
213
272
|
const plan = findPlan(cfg, scheduleId);
|
|
@@ -255,6 +314,13 @@ class SchedulerCommand extends Command_1.BaseCommand {
|
|
|
255
314
|
// attempts are exhausted, which does not block exit), so this
|
|
256
315
|
// cannot hold the machine open indefinitely.
|
|
257
316
|
pendingOutbox: outbox.pending,
|
|
317
|
+
// Second belt: never exit underneath an in-flight slot execution
|
|
318
|
+
// even if a durable row ever looks a beat early (§idle-inflight).
|
|
319
|
+
activeExecutions: runtime.activeExecutionCount(),
|
|
320
|
+
// Third belt: work parked waiting for resource capacity is
|
|
321
|
+
// unfinished work (§resource-governance) — a queued run must not
|
|
322
|
+
// let the machine stop before it runs.
|
|
323
|
+
waitingForResource: manager.waitingWorkCount(),
|
|
258
324
|
};
|
|
259
325
|
},
|
|
260
326
|
idleGraceMs: scheduling.idleGraceMs,
|
|
@@ -41,6 +41,23 @@ export interface IdleSnapshot {
|
|
|
41
41
|
processingOutbox: number;
|
|
42
42
|
/** Undelivered outbox rows (pending, or waiting on a bounded retry backoff). */
|
|
43
43
|
pendingOutbox: number;
|
|
44
|
+
/**
|
|
45
|
+
* In-process slot executions currently awaiting async work (candidate scan,
|
|
46
|
+
* download, delivery). This is a SECOND belt behind the durable ledger: even
|
|
47
|
+
* if a DB row is an instant ahead of its awaiting Promise (or a mis-write
|
|
48
|
+
* ever terminalized a row early), the idle detector must not exit underneath
|
|
49
|
+
* a live execution. In-memory only — a process crash still resumes from the
|
|
50
|
+
* durable slot/outbox rows.
|
|
51
|
+
*/
|
|
52
|
+
activeExecutions: number;
|
|
53
|
+
/**
|
|
54
|
+
* Work items parked in the resource admission queue (§resource-governance).
|
|
55
|
+
* Waiting for capacity is UNFINISHED work, not idleness: a queued run has a
|
|
56
|
+
* durable slot row (counted above) but may not have reached `claimRunLease`
|
|
57
|
+
* yet in internal mode, so this explicit belt prevents the worker from
|
|
58
|
+
* shutting down while a run is still waiting to start.
|
|
59
|
+
*/
|
|
60
|
+
waitingForResource: number;
|
|
44
61
|
}
|
|
45
62
|
/** True only when the worker has no work of any kind left. */
|
|
46
63
|
export declare function isIdle(snapshot: IdleSnapshot): boolean;
|
|
@@ -38,7 +38,9 @@ const logger_1 = require("../logger");
|
|
|
38
38
|
function isIdle(snapshot) {
|
|
39
39
|
return (snapshot.activeSlots === 0 &&
|
|
40
40
|
snapshot.processingOutbox === 0 &&
|
|
41
|
-
snapshot.pendingOutbox === 0
|
|
41
|
+
snapshot.pendingOutbox === 0 &&
|
|
42
|
+
snapshot.activeExecutions === 0 &&
|
|
43
|
+
snapshot.waitingForResource === 0);
|
|
42
44
|
}
|
|
43
45
|
/** Default idle grace: long enough to merge two schedules ~10 minutes apart. */
|
|
44
46
|
exports.DEFAULT_IDLE_GRACE_MS = 10 * 60 * 1000;
|
|
@@ -100,6 +100,14 @@ export interface SchedulerRuntime {
|
|
|
100
100
|
notifyScheduleFailure(snapshot: StandaloneConfig, schedule: ScheduleConfig, failure: JobFailure): Promise<void>;
|
|
101
101
|
/** Start the independent outbox pump (long-running daemon). */
|
|
102
102
|
startOutboxWorker(): void;
|
|
103
|
+
/**
|
|
104
|
+
* Number of slot executions currently awaiting async work in THIS process
|
|
105
|
+
* (candidate scan / download / delivery). Bounded second belt for the idle
|
|
106
|
+
* lifecycle: exit is only allowed when this is 0 AND the durable ledger is
|
|
107
|
+
* empty (§idle-inflight). In-memory only; crash recovery still rides the
|
|
108
|
+
* durable slot/outbox rows.
|
|
109
|
+
*/
|
|
110
|
+
activeExecutionCount(): number;
|
|
103
111
|
/** Drain due outbox rows once (run-once / watchdog wake). */
|
|
104
112
|
drainOutbox(): Promise<{
|
|
105
113
|
processed: number;
|
|
@@ -58,6 +58,7 @@ const DeliveryDispatcher_1 = require("../delivery/DeliveryDispatcher");
|
|
|
58
58
|
const token_maintenance_1 = require("../utils/token-maintenance");
|
|
59
59
|
const schedules_1 = require("../scheduler/schedules");
|
|
60
60
|
const SlotCoordinator_1 = require("../scheduler/SlotCoordinator");
|
|
61
|
+
const RecoveryPolicy_1 = require("../scheduler/RecoveryPolicy");
|
|
61
62
|
const DeliveryLedgerPort_1 = require("../delivery/DeliveryLedgerPort");
|
|
62
63
|
const OutboxWorker_1 = require("../delivery/OutboxWorker");
|
|
63
64
|
const settleDeliveryTerminal_1 = require("../delivery/settleDeliveryTerminal");
|
|
@@ -275,6 +276,7 @@ async function createSchedulerRuntime(configPathArg) {
|
|
|
275
276
|
notificationPolicy.noteTerminalRefetchCell(delivery.slotId, delivery.targetId);
|
|
276
277
|
},
|
|
277
278
|
});
|
|
279
|
+
let activeExecutions = 0;
|
|
278
280
|
const runJob = async (snapshot, schedule, options = {}) => {
|
|
279
281
|
const { onlyTarget, adhoc = false, slot: providedSlot } = options;
|
|
280
282
|
// A scheduled run is cron by default; an HTTP/manual trigger passes its own.
|
|
@@ -285,6 +287,20 @@ async function createSchedulerRuntime(configPathArg) {
|
|
|
285
287
|
// "重抓/换一张" 只重跑产生该审核的那一个 target。
|
|
286
288
|
targets = targets.filter((t) => t.id === onlyTarget);
|
|
287
289
|
}
|
|
290
|
+
// Occurrence-scoped acquisition policy (§recovery-policy). A manual recovery
|
|
291
|
+
// slot carries a server-defined preset; `relaxed` widens only SOFT criteria
|
|
292
|
+
// (search range / candidate count / language window) for THIS occurrence.
|
|
293
|
+
// The global config is never written, so future schedules are unaffected.
|
|
294
|
+
const recoveryMode = providedSlot?.recoveryMode;
|
|
295
|
+
if (recoveryMode && recoveryMode !== 'normal') {
|
|
296
|
+
targets = targets.map((target) => (0, RecoveryPolicy_1.applyAcquisitionPolicy)(target, recoveryMode));
|
|
297
|
+
logger_1.logger.info('Manual recovery policy applied to this occurrence', {
|
|
298
|
+
scheduleId: schedule.id,
|
|
299
|
+
slot: providedSlot?.slotId,
|
|
300
|
+
recoveryMode,
|
|
301
|
+
targets: targets.map((t) => t.id),
|
|
302
|
+
});
|
|
303
|
+
}
|
|
288
304
|
if (targets.length === 0) {
|
|
289
305
|
logger_1.logger.warn('Scheduled plan has no selected targets; skipping', {
|
|
290
306
|
scheduleId: schedule.id,
|
|
@@ -546,107 +562,117 @@ async function createSchedulerRuntime(configPathArg) {
|
|
|
546
562
|
});
|
|
547
563
|
await new Promise((resolve) => setTimeout(resolve, runtimeConfig.initialDelay));
|
|
548
564
|
}
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
slot: slotCtx?.slotId ?? '(ad-hoc)',
|
|
553
|
-
trigger: triggerSource,
|
|
554
|
-
targets: runTargets.map((target) => target.id ?? target.tag ?? target.filterTag ?? target.type),
|
|
555
|
-
});
|
|
556
|
-
logger_1.logger.info('='.repeat(60));
|
|
557
|
-
const startTime = Date.now();
|
|
558
|
-
let allTargetsFailed;
|
|
559
|
-
let releaseLease = () => undefined;
|
|
560
|
-
if (slotCtx && varReleaseLease)
|
|
561
|
-
releaseLease = varReleaseLease;
|
|
565
|
+
// Execution belt (§idle-inflight): this run is live until releaseLease
|
|
566
|
+
// completes. The idle detector must see it even if a durable row lags.
|
|
567
|
+
activeExecutions += 1;
|
|
562
568
|
try {
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
569
|
+
logger_1.logger.info('='.repeat(60));
|
|
570
|
+
logger_1.logger.info('Starting scheduled Pixiv download plan', {
|
|
571
|
+
scheduleId: schedule.id,
|
|
572
|
+
slot: slotCtx?.slotId ?? '(ad-hoc)',
|
|
573
|
+
trigger: triggerSource,
|
|
574
|
+
targets: runTargets.map((target) => target.id ?? target.tag ?? target.filterTag ?? target.type),
|
|
575
|
+
});
|
|
576
|
+
logger_1.logger.info('='.repeat(60));
|
|
577
|
+
const startTime = Date.now();
|
|
578
|
+
let allTargetsFailed;
|
|
579
|
+
let releaseLease = () => undefined;
|
|
580
|
+
if (slotCtx && varReleaseLease)
|
|
581
|
+
releaseLease = varReleaseLease;
|
|
582
|
+
try {
|
|
583
|
+
// Candidate fallback passes (§schedule-recovery): after each pass, cells
|
|
584
|
+
// still mid-fallback (advanced stages) are re-selected with expanded,
|
|
585
|
+
// still-bounded scan limits. Successful sibling cells are NEVER re-run:
|
|
586
|
+
// pendingTargets only returns unconverged cells of this slot.
|
|
587
|
+
for (let pass = 0;; pass += 1) {
|
|
588
|
+
await downloadManager.runAllTargets();
|
|
589
|
+
if (!slotCtx || pass >= maxFallbackStages - 1)
|
|
590
|
+
break;
|
|
591
|
+
const pendingFallback = coordinator
|
|
592
|
+
.pendingTargets(slotCtx.slotId, targets)
|
|
593
|
+
.filter((p) => p.cell &&
|
|
594
|
+
p.cell.fallback_stage > 0 &&
|
|
595
|
+
p.cell.fallback_stage < maxFallbackStages);
|
|
596
|
+
if (pendingFallback.length === 0)
|
|
597
|
+
break;
|
|
598
|
+
downloadManager = buildManager(boostedTargets(pendingFallback.map((p) => p.target)));
|
|
599
|
+
activeDownloadManager = downloadManager;
|
|
600
|
+
await downloadManager.initialise();
|
|
601
|
+
}
|
|
581
602
|
}
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
+
catch (error) {
|
|
604
|
+
if (!slotCtx || !(error instanceof Error) || !/^All \d+ target\(s\) failed\./.test(error.message)) {
|
|
605
|
+
// Abnormal abort. Two very different situations land here, and treating
|
|
606
|
+
// them as one is what produced the production loop:
|
|
607
|
+
//
|
|
608
|
+
// - this process cancelled itself (scheduler timeout / watchdog) and is
|
|
609
|
+
// still alive. It will never touch the Slot again, so releasing the
|
|
610
|
+
// lease while leaving the status `running` makes the Slot look exactly
|
|
611
|
+
// like a crashed worker: `recoverableSlots()` matches a NULL lease on
|
|
612
|
+
// purpose (its "recorded but never claimed" case), so the recovery
|
|
613
|
+
// sweep re-dispatches the SAME occurrence on every tick, forever.
|
|
614
|
+
// Terminalise it instead: the occurrence is finished, and failed.
|
|
615
|
+
//
|
|
616
|
+
// - the process is going away (`shutdown`). Deliberately leave the Slot
|
|
617
|
+
// non-terminal so recovery resumes the same occurrence after restart.
|
|
618
|
+
if (slotCtx && shouldTerminaliseAbortedSlot(activeAbortOrigin, slotAbandoned)) {
|
|
619
|
+
coordinator.finish(slotCtx, schedule, targets);
|
|
620
|
+
for (const target of targets)
|
|
621
|
+
if (target.id)
|
|
622
|
+
notificationPolicy.noteTerminalRefetchCell(slotCtx.slotId, target.id);
|
|
623
|
+
}
|
|
624
|
+
// Hand the lease back either way: a terminal Slot cannot be re-dispatched,
|
|
625
|
+
// and a non-terminal one (shutdown) must not wait for its TTL to expire.
|
|
626
|
+
releaseLease?.();
|
|
627
|
+
throw error;
|
|
628
|
+
}
|
|
629
|
+
// Scheduled Slots treat terminal target failures (failed/no_candidate) as
|
|
630
|
+
// a finished partial/failed aggregate. Finish before returning so the HTTP
|
|
631
|
+
// trigger can report the durable state instead of 500 and never roll up.
|
|
632
|
+
allTargetsFailed = error;
|
|
633
|
+
}
|
|
634
|
+
finally {
|
|
635
|
+
if (activeDownloadManager === downloadManager)
|
|
636
|
+
activeDownloadManager = null;
|
|
637
|
+
}
|
|
638
|
+
const duration = Math.round((Date.now() - startTime) / 1000);
|
|
639
|
+
if (slotCtx && !slotAbandoned) {
|
|
640
|
+
const summary = coordinator.finish(slotCtx, schedule, targets);
|
|
641
|
+
for (const target of targets)
|
|
642
|
+
if (target.id)
|
|
643
|
+
notificationPolicy.noteTerminalRefetchCell(slotCtx.slotId, target.id);
|
|
644
|
+
// Terminal summaries are for SCHEDULED occurrences (P0-B). Remote manual
|
|
645
|
+
// replacements already report through their own verdict channel
|
|
646
|
+
// (refetchOutcomeUrl); a schedule summary would double-notify the group.
|
|
647
|
+
if (!slotCtx.manualRequestId) {
|
|
648
|
+
notificationPolicy.sendSlotSummary(slotCtx, schedule, summary.cells.map((c) => {
|
|
649
|
+
const t = targets.find((x) => x.id === c.targetId);
|
|
650
|
+
return {
|
|
651
|
+
targetId: c.targetId,
|
|
652
|
+
label: c.targetId,
|
|
653
|
+
workType: t?.type ?? 'unknown',
|
|
654
|
+
status: c.status,
|
|
655
|
+
workId: c.workId,
|
|
656
|
+
error: c.error ?? null,
|
|
657
|
+
terminal_reason_code: c.terminalReasonCode ?? null,
|
|
658
|
+
reason: c.terminalReasonMessage ?? null,
|
|
659
|
+
};
|
|
660
|
+
}));
|
|
603
661
|
}
|
|
604
|
-
|
|
605
|
-
// and a non-terminal one (shutdown) must not wait for its TTL to expire.
|
|
606
|
-
releaseLease?.();
|
|
607
|
-
throw error;
|
|
662
|
+
releaseLease();
|
|
608
663
|
}
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
664
|
+
if (!slotCtx && allTargetsFailed)
|
|
665
|
+
throw allTargetsFailed;
|
|
666
|
+
logger_1.logger.info('='.repeat(60));
|
|
667
|
+
logger_1.logger.info(`Scheduled download plan finished (took ${duration}s)`, {
|
|
668
|
+
scheduleId: schedule.id,
|
|
669
|
+
slot: slotCtx?.slotId ?? '(ad-hoc)',
|
|
670
|
+
});
|
|
671
|
+
logger_1.logger.info('='.repeat(60));
|
|
613
672
|
}
|
|
614
673
|
finally {
|
|
615
|
-
|
|
616
|
-
activeDownloadManager = null;
|
|
617
|
-
}
|
|
618
|
-
const duration = Math.round((Date.now() - startTime) / 1000);
|
|
619
|
-
if (slotCtx && !slotAbandoned) {
|
|
620
|
-
const summary = coordinator.finish(slotCtx, schedule, targets);
|
|
621
|
-
for (const target of targets)
|
|
622
|
-
if (target.id)
|
|
623
|
-
notificationPolicy.noteTerminalRefetchCell(slotCtx.slotId, target.id);
|
|
624
|
-
// Terminal summaries are for SCHEDULED occurrences (P0-B). Remote manual
|
|
625
|
-
// replacements already report through their own verdict channel
|
|
626
|
-
// (refetchOutcomeUrl); a schedule summary would double-notify the group.
|
|
627
|
-
if (!slotCtx.manualRequestId) {
|
|
628
|
-
notificationPolicy.sendSlotSummary(slotCtx, schedule, summary.cells.map((c) => {
|
|
629
|
-
const t = targets.find((x) => x.id === c.targetId);
|
|
630
|
-
return {
|
|
631
|
-
targetId: c.targetId,
|
|
632
|
-
label: c.targetId,
|
|
633
|
-
workType: t?.type ?? 'unknown',
|
|
634
|
-
status: c.status,
|
|
635
|
-
workId: c.workId,
|
|
636
|
-
error: c.error ?? null,
|
|
637
|
-
};
|
|
638
|
-
}));
|
|
639
|
-
}
|
|
640
|
-
releaseLease();
|
|
674
|
+
activeExecutions -= 1;
|
|
641
675
|
}
|
|
642
|
-
if (!slotCtx && allTargetsFailed)
|
|
643
|
-
throw allTargetsFailed;
|
|
644
|
-
logger_1.logger.info('='.repeat(60));
|
|
645
|
-
logger_1.logger.info(`Scheduled download plan finished (took ${duration}s)`, {
|
|
646
|
-
scheduleId: schedule.id,
|
|
647
|
-
slot: slotCtx?.slotId ?? '(ad-hoc)',
|
|
648
|
-
});
|
|
649
|
-
logger_1.logger.info('='.repeat(60));
|
|
650
676
|
};
|
|
651
677
|
const cancelActive = (reason, origin = 'timeout') => {
|
|
652
678
|
activeAbortOrigin = origin;
|
|
@@ -680,6 +706,7 @@ async function createSchedulerRuntime(configPathArg) {
|
|
|
680
706
|
runJob,
|
|
681
707
|
cancelActive,
|
|
682
708
|
abandonActiveRun,
|
|
709
|
+
activeExecutionCount: () => activeExecutions,
|
|
683
710
|
startOutboxWorker: () => outboxWorker.start(),
|
|
684
711
|
drainOutbox: () => outboxWorker.drainOnce(),
|
|
685
712
|
notifyScheduleFailure: (snapshot, schedule, failure) => notifyScheduleFailure(snapshot, database, schedule, failure),
|
package/dist/config/types.d.ts
CHANGED
|
@@ -279,6 +279,25 @@ export interface PixivCredentialConfig {
|
|
|
279
279
|
deviceToken: string;
|
|
280
280
|
refreshToken: string;
|
|
281
281
|
userAgent: string;
|
|
282
|
+
/**
|
|
283
|
+
* Stable internal identity of this credential profile (§resource-governance).
|
|
284
|
+
* It is the RESOURCE identity used for admission keys (`pixiv-account:<id>`),
|
|
285
|
+
* never a token/cookie and never a bot/schedule/target name. Default 'default'.
|
|
286
|
+
*/
|
|
287
|
+
accountId?: string;
|
|
288
|
+
}
|
|
289
|
+
/**
|
|
290
|
+
* Operator-level resource capacity configuration (§resource-governance).
|
|
291
|
+
* Concurrency is scoped by the real constrained resource, not by bot/schedule/
|
|
292
|
+
* target. Current production resource: the single Pixiv account profile, with
|
|
293
|
+
* `maxConcurrency: 1` recommended. This is deployment config — never exposed
|
|
294
|
+
* to ordinary manager UI.
|
|
295
|
+
*/
|
|
296
|
+
export interface ResourceGovernanceConfig {
|
|
297
|
+
/** Per-Pixiv-account-profile capacity. Keyed by `pixiv.accountId`. */
|
|
298
|
+
pixivAccounts?: Record<string, {
|
|
299
|
+
maxConcurrency?: number;
|
|
300
|
+
}>;
|
|
282
301
|
}
|
|
283
302
|
export interface NetworkConfig {
|
|
284
303
|
/**
|
|
@@ -473,6 +492,13 @@ export interface SchedulerRuntimeConfig {
|
|
|
473
492
|
* resumes from the same durable Slot/outbox rows. Default: 10800000ms (3h).
|
|
474
493
|
*/
|
|
475
494
|
maxLifetimeMs?: number;
|
|
495
|
+
/**
|
|
496
|
+
* Operator-level resource capacity (§resource-governance). Concurrency is
|
|
497
|
+
* scoped by the real constrained resource (e.g. the shared Pixiv account),
|
|
498
|
+
* never by bot/schedule/target. Waiting for capacity is a normal state, not
|
|
499
|
+
* an execution failure. Not exposed to ordinary manager UI.
|
|
500
|
+
*/
|
|
501
|
+
resourceGovernance?: ResourceGovernanceConfig;
|
|
476
502
|
/**
|
|
477
503
|
* External-mode HTTP trigger settings. Ignored in internal mode.
|
|
478
504
|
*/
|
|
@@ -473,6 +473,27 @@ function validateConfig(config, location, databasePath) {
|
|
|
473
473
|
errors.push('schedulerRuntime.trigger.graceMinutes: Must be a positive integer (minutes)');
|
|
474
474
|
}
|
|
475
475
|
}
|
|
476
|
+
const rg = rt.resourceGovernance;
|
|
477
|
+
if (rg) {
|
|
478
|
+
if (rg.pixivAccounts !== undefined && (typeof rg.pixivAccounts !== 'object' || rg.pixivAccounts === null || Array.isArray(rg.pixivAccounts))) {
|
|
479
|
+
errors.push('schedulerRuntime.resourceGovernance.pixivAccounts: Must be an object keyed by pixiv account id');
|
|
480
|
+
}
|
|
481
|
+
else if (rg.pixivAccounts) {
|
|
482
|
+
for (const [accountId, profile] of Object.entries(rg.pixivAccounts)) {
|
|
483
|
+
if (!accountId.trim()) {
|
|
484
|
+
errors.push('schedulerRuntime.resourceGovernance.pixivAccounts: account ids must be non-empty');
|
|
485
|
+
}
|
|
486
|
+
if (profile === null || typeof profile !== 'object' || Array.isArray(profile)) {
|
|
487
|
+
errors.push(`schedulerRuntime.resourceGovernance.pixivAccounts.${accountId}: Must be an object`);
|
|
488
|
+
continue;
|
|
489
|
+
}
|
|
490
|
+
const max = profile.maxConcurrency;
|
|
491
|
+
if (max !== undefined && (!Number.isInteger(max) || max < 1 || max > 16)) {
|
|
492
|
+
errors.push(`schedulerRuntime.resourceGovernance.pixivAccounts.${accountId}.maxConcurrency: Must be an integer between 1 and 16`);
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
}
|
|
476
497
|
}
|
|
477
498
|
// Validate download config
|
|
478
499
|
if (config.download) {
|
package/dist/delivery/types.d.ts
CHANGED
|
@@ -122,11 +122,26 @@ export interface DeliveryNotificationRequest {
|
|
|
122
122
|
scheduleId: string;
|
|
123
123
|
slotId: string;
|
|
124
124
|
status: 'success' | 'partial' | 'failed';
|
|
125
|
+
/**
|
|
126
|
+
* Present when this terminal outcome belongs to a MANUAL RECOVERY run
|
|
127
|
+
* (§manual-recovery): the request id and the policy preset used. Lets the
|
|
128
|
+
* receiving service render "已恢复" instead of the daily summary.
|
|
129
|
+
*/
|
|
130
|
+
recovery?: {
|
|
131
|
+
mode: 'normal' | 'relaxed';
|
|
132
|
+
requestId: string;
|
|
133
|
+
};
|
|
125
134
|
targets?: Array<{
|
|
126
135
|
targetId: string;
|
|
127
136
|
workType: string;
|
|
128
137
|
status: string;
|
|
129
138
|
workId?: string | null;
|
|
139
|
+
/** Raw cell error (request-level observability; NOT user-safe text). */
|
|
140
|
+
error?: string | null;
|
|
141
|
+
/** Normalized terminal failure code (§terminal-reason). */
|
|
142
|
+
terminal_reason_code?: string | null;
|
|
143
|
+
/** User-facing business reason message for the failure. */
|
|
144
|
+
reason?: string | null;
|
|
130
145
|
}>;
|
|
131
146
|
};
|
|
132
147
|
}
|
|
@@ -44,6 +44,8 @@ export declare class NotificationPolicy {
|
|
|
44
44
|
status: string;
|
|
45
45
|
workId: string | null;
|
|
46
46
|
error: string | null;
|
|
47
|
+
terminal_reason_code?: string | null;
|
|
48
|
+
reason?: string | null;
|
|
47
49
|
}>): void;
|
|
48
50
|
/**
|
|
49
51
|
* Delivery targets whose HTTP target declares the given outcome URL.
|
|
@@ -88,7 +88,13 @@ class NotificationPolicy {
|
|
|
88
88
|
(r.workId ? ` #${r.workId}` : '') +
|
|
89
89
|
(r.status === 'delivery_pending' ? ' 投递中' : '') +
|
|
90
90
|
(r.status === 'no_candidate' ? ' 无候选' : '') +
|
|
91
|
-
|
|
91
|
+
// First-level cause, business language only: the durable normalized
|
|
92
|
+
// reason replaces any raw error text (§terminal-reason).
|
|
93
|
+
((r.status === 'failed' || r.status === 'no_candidate' || r.status === 'duplicate') && r.reason
|
|
94
|
+
? ` ${r.reason.slice(0, 80)}`
|
|
95
|
+
: r.status === 'failed' && r.error
|
|
96
|
+
? ` ${r.error.slice(0, 80)}`
|
|
97
|
+
: ''));
|
|
92
98
|
const submitted = rows.filter((r) => r.status === 'submitted').length;
|
|
93
99
|
const text = [
|
|
94
100
|
`${schedule.name?.trim() || schedule.id} · ${slot.occurrenceLabel}`,
|
|
@@ -102,11 +108,22 @@ class NotificationPolicy {
|
|
|
102
108
|
scheduleId: schedule.id,
|
|
103
109
|
slotId: slot.slotId,
|
|
104
110
|
status: outcomeStatus,
|
|
111
|
+
...(slot.recoveryRequestId
|
|
112
|
+
? {
|
|
113
|
+
recovery: {
|
|
114
|
+
mode: slot.recoveryMode ?? 'normal',
|
|
115
|
+
requestId: slot.recoveryRequestId,
|
|
116
|
+
},
|
|
117
|
+
}
|
|
118
|
+
: {}),
|
|
105
119
|
targets: rows.map((r) => ({
|
|
106
120
|
targetId: r.targetId,
|
|
107
121
|
workType: r.workType,
|
|
108
122
|
status: r.status,
|
|
109
123
|
workId: r.workId,
|
|
124
|
+
error: r.error,
|
|
125
|
+
terminal_reason_code: r.terminal_reason_code ?? null,
|
|
126
|
+
reason: r.reason ?? null,
|
|
110
127
|
})),
|
|
111
128
|
});
|
|
112
129
|
}
|
package/dist/package.json
CHANGED
|
@@ -34,7 +34,9 @@ export declare const RECOVERY_INTERVAL_MS: number;
|
|
|
34
34
|
/**
|
|
35
35
|
* Hosts many cron plans in one process. A validated config snapshot replaces
|
|
36
36
|
* the complete cron table at once; invalid updates leave the previous table
|
|
37
|
-
* running. All jobs
|
|
37
|
+
* running. All jobs that consume the same constrained resource (today: the
|
|
38
|
+
* shared Pixiv account) pass through ONE bounded resource admission; work for
|
|
39
|
+
* different resource identities runs concurrently.
|
|
38
40
|
*/
|
|
39
41
|
export declare class MultiScheduleManager {
|
|
40
42
|
private readonly options;
|
|
@@ -46,6 +48,15 @@ export declare class MultiScheduleManager {
|
|
|
46
48
|
private watching;
|
|
47
49
|
private readonly admission;
|
|
48
50
|
constructor(options: MultiScheduleManagerOptions);
|
|
51
|
+
/**
|
|
52
|
+
* Deduplicated, log-safe observable: how many work items are parked waiting
|
|
53
|
+
* for any resource. Used by the idle lifecycle — waiting work is unfinished
|
|
54
|
+
* work and must prevent premature shutdown (§waiting-is-not-idle).
|
|
55
|
+
*/
|
|
56
|
+
waitingWorkCount(): number;
|
|
57
|
+
/** The resource key a plan's work consumes (resource identity, never a bot/
|
|
58
|
+
* schedule/target name). All Pixiv-consuming work shares the account id. */
|
|
59
|
+
private resourceKeyForPlan;
|
|
49
60
|
start(initialConfig?: StandaloneConfig): ConfigReloadResult;
|
|
50
61
|
private startRecoveryLoop;
|
|
51
62
|
/**
|