pixivflow 2.20.3 → 2.20.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +7 -3
- package/README.md +6 -5
- package/dist/commands/SchedulerCommand.js +3 -0
- package/dist/commands/SchedulerIdleLifecycle.d.ts +9 -0
- package/dist/commands/SchedulerIdleLifecycle.js +2 -1
- package/dist/commands/scheduler-runtime.d.ts +8 -0
- package/dist/commands/scheduler-runtime.js +111 -94
- package/dist/delivery/OutboxWorker.d.ts +3 -0
- package/dist/delivery/OutboxWorker.js +4 -0
- package/dist/notification/NotificationPolicy.d.ts +2 -0
- package/dist/notification/NotificationPolicy.js +31 -5
- package/dist/package.json +1 -1
- package/dist/storage/repositories/SlotRepository.d.ts +2 -0
- package/dist/storage/repositories/SlotRepository.js +12 -0
- package/dist/version.js +1 -1
- package/dist/webui/package.json +1 -1
- package/package.json +1 -1
package/README.en.md
CHANGED
|
@@ -22,7 +22,9 @@ pixivflow --help
|
|
|
22
22
|
```
|
|
23
23
|
|
|
24
24
|
For servers, prefer the Docker Compose setup described in
|
|
25
|
-
[DOCKER.md](docs/DOCKER.md). To
|
|
25
|
+
[DOCKER.md](docs/DOCKER.md). To compose PixivFlow with TelePost, use the optional
|
|
26
|
+
[pixivflow-telepost-deploy](https://github.com/redtidev1918/pixivflow-telepost-deploy)
|
|
27
|
+
deployment and operations kit. To build from source:
|
|
26
28
|
|
|
27
29
|
```bash
|
|
28
30
|
git clone https://github.com/redtidev1918/PixivFlow.git
|
|
@@ -92,8 +94,10 @@ conflict.
|
|
|
92
94
|
|
|
93
95
|
Each target can use `storageMode: "persistent"` (the default, keep files) or
|
|
94
96
|
`storageMode: "cache"` (send files to a named delivery target and delete them
|
|
95
|
-
only after success). The delivery layer is service-independent;
|
|
96
|
-
|
|
97
|
+
only after success). The delivery layer is service-independent;
|
|
98
|
+
[TelePost](https://github.com/redtidev1918/TelePost) and
|
|
99
|
+
[telepress](https://github.com/redtidev1918/telepress) are example downstreams,
|
|
100
|
+
not dependencies. This example merely translates an HTTP multipart submission API into configuration:
|
|
97
101
|
|
|
98
102
|
```json
|
|
99
103
|
{
|
package/README.md
CHANGED
|
@@ -22,10 +22,9 @@ npm install -g pixivflow
|
|
|
22
22
|
pixivflow --help
|
|
23
23
|
```
|
|
24
24
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
—— 一套配置,支持国内/海外、有/无公网 IP、VPS/Fly.io 任意场景。
|
|
25
|
+
服务器部署见 [DOCKER.md](docs/DOCKER.md)。需要把 PixivFlow 与 TelePost 组合运行时,再使用
|
|
26
|
+
[pixivflow-telepost-deploy](https://github.com/redtidev1918/pixivflow-telepost-deploy);
|
|
27
|
+
它是可选的部署与运维套件。
|
|
29
28
|
从源码构建:
|
|
30
29
|
|
|
31
30
|
```bash
|
|
@@ -111,7 +110,9 @@ pixivflow scheduler # 按 cron 配置长期挂机自动收集
|
|
|
111
110
|
- **`cache`**:下载后投给一个「交付目标」(比如投稿机器人),对方确认收到后才删本地文件,省磁盘。
|
|
112
111
|
|
|
113
112
|
「交付目标」就是一段配置:告诉 PixivFlow 把文件 POST 到哪个地址、带哪些字段。
|
|
114
|
-
|
|
113
|
+
它不绑定具体服务,可指向任意兼容的 HTTP 接口;
|
|
114
|
+
[TelePost](https://github.com/redtidev1918/TelePost) 与
|
|
115
|
+
[telepress](https://github.com/redtidev1918/telepress) 只是示例下游。示例:
|
|
115
116
|
|
|
116
117
|
```json
|
|
117
118
|
{
|
|
@@ -255,6 +255,9 @@ class SchedulerCommand extends Command_1.BaseCommand {
|
|
|
255
255
|
// attempts are exhausted, which does not block exit), so this
|
|
256
256
|
// cannot hold the machine open indefinitely.
|
|
257
257
|
pendingOutbox: outbox.pending,
|
|
258
|
+
// Second belt: never exit underneath an in-flight slot execution
|
|
259
|
+
// even if a durable row ever looks a beat early (§idle-inflight).
|
|
260
|
+
activeExecutions: runtime.activeExecutionCount(),
|
|
258
261
|
};
|
|
259
262
|
},
|
|
260
263
|
idleGraceMs: scheduling.idleGraceMs,
|
|
@@ -41,6 +41,15 @@ export interface IdleSnapshot {
|
|
|
41
41
|
processingOutbox: number;
|
|
42
42
|
/** Undelivered outbox rows (pending, or waiting on a bounded retry backoff). */
|
|
43
43
|
pendingOutbox: number;
|
|
44
|
+
/**
|
|
45
|
+
* In-process slot executions currently awaiting async work (candidate scan,
|
|
46
|
+
* download, delivery). This is a SECOND belt behind the durable ledger: even
|
|
47
|
+
* if a DB row is an instant ahead of its awaiting Promise (or a mis-write
|
|
48
|
+
* ever terminalized a row early), the idle detector must not exit underneath
|
|
49
|
+
* a live execution. In-memory only — a process crash still resumes from the
|
|
50
|
+
* durable slot/outbox rows.
|
|
51
|
+
*/
|
|
52
|
+
activeExecutions: number;
|
|
44
53
|
}
|
|
45
54
|
/** True only when the worker has no work of any kind left. */
|
|
46
55
|
export declare function isIdle(snapshot: IdleSnapshot): boolean;
|
|
@@ -38,7 +38,8 @@ const logger_1 = require("../logger");
|
|
|
38
38
|
function isIdle(snapshot) {
|
|
39
39
|
return (snapshot.activeSlots === 0 &&
|
|
40
40
|
snapshot.processingOutbox === 0 &&
|
|
41
|
-
snapshot.pendingOutbox === 0
|
|
41
|
+
snapshot.pendingOutbox === 0 &&
|
|
42
|
+
snapshot.activeExecutions === 0);
|
|
42
43
|
}
|
|
43
44
|
/** Default idle grace: long enough to merge two schedules ~10 minutes apart. */
|
|
44
45
|
exports.DEFAULT_IDLE_GRACE_MS = 10 * 60 * 1000;
|
|
@@ -100,6 +100,14 @@ export interface SchedulerRuntime {
|
|
|
100
100
|
notifyScheduleFailure(snapshot: StandaloneConfig, schedule: ScheduleConfig, failure: JobFailure): Promise<void>;
|
|
101
101
|
/** Start the independent outbox pump (long-running daemon). */
|
|
102
102
|
startOutboxWorker(): void;
|
|
103
|
+
/**
|
|
104
|
+
* Number of slot executions currently awaiting async work in THIS process
|
|
105
|
+
* (candidate scan / download / delivery). Bounded second belt for the idle
|
|
106
|
+
* lifecycle: exit is only allowed when this is 0 AND the durable ledger is
|
|
107
|
+
* empty (§idle-inflight). In-memory only; crash recovery still rides the
|
|
108
|
+
* durable slot/outbox rows.
|
|
109
|
+
*/
|
|
110
|
+
activeExecutionCount(): number;
|
|
103
111
|
/** Drain due outbox rows once (run-once / watchdog wake). */
|
|
104
112
|
drainOutbox(): Promise<{
|
|
105
113
|
processed: number;
|
|
@@ -252,6 +252,7 @@ async function createSchedulerRuntime(configPathArg) {
|
|
|
252
252
|
const deliveryDispatcher = new DeliveryDispatcher_1.DeliveryDispatcher(config.delivery, buildProxyUrl(config.network));
|
|
253
253
|
const notificationPolicy = new NotificationPolicy_1.NotificationPolicy(database, config);
|
|
254
254
|
const outboxWorker = new OutboxWorker_1.OutboxWorker(database, deliveryDispatcher, {
|
|
255
|
+
beforeDrain: () => notificationPolicy.reconcileScheduleSummaries(),
|
|
255
256
|
retryBaseMs: config.delivery?.outboxRetryBaseMs,
|
|
256
257
|
retryMaxMs: config.delivery?.outboxRetryMaxMs,
|
|
257
258
|
// A confirmed ACK settles the owning Slot cell (submitted / duplicate /
|
|
@@ -274,6 +275,7 @@ async function createSchedulerRuntime(configPathArg) {
|
|
|
274
275
|
notificationPolicy.noteTerminalRefetchCell(delivery.slotId, delivery.targetId);
|
|
275
276
|
},
|
|
276
277
|
});
|
|
278
|
+
let activeExecutions = 0;
|
|
277
279
|
const runJob = async (snapshot, schedule, options = {}) => {
|
|
278
280
|
const { onlyTarget, adhoc = false, slot: providedSlot } = options;
|
|
279
281
|
// A scheduled run is cron by default; an HTTP/manual trigger passes its own.
|
|
@@ -462,12 +464,13 @@ async function createSchedulerRuntime(configPathArg) {
|
|
|
462
464
|
// as a degraded (partial) terminal result.
|
|
463
465
|
const maxFallbackStages = Math.max(0, Math.min(10, Number(runtimeConfig.download?.maxFallbackStages ?? 3)));
|
|
464
466
|
const boostScanLimit = fallbackScanLimit;
|
|
465
|
-
const boostedTargets = (list
|
|
466
|
-
?
|
|
467
|
-
|
|
467
|
+
const boostedTargets = (list) => list.map((t) => {
|
|
468
|
+
const stage = scheduleSlot && t.id ? coordinator.cellFallbackStage(scheduleSlot.slotId, t.id) : 0;
|
|
469
|
+
return stage === 0 ? t : {
|
|
468
470
|
...t,
|
|
469
471
|
candidateScanLimit: boostScanLimit(t.candidateScanLimit ?? runtimeConfig.download?.candidateScanLimit, stage),
|
|
470
|
-
}
|
|
472
|
+
};
|
|
473
|
+
});
|
|
471
474
|
const buildManager = (list) => {
|
|
472
475
|
const scoped = {
|
|
473
476
|
...runtimeConfig,
|
|
@@ -534,7 +537,7 @@ async function createSchedulerRuntime(configPathArg) {
|
|
|
534
537
|
notificationPolicy.noteRefetchOutcome(scheduleSlot, schedule, target, scheduleSlot.manualRequestId, outcome);
|
|
535
538
|
}
|
|
536
539
|
};
|
|
537
|
-
let downloadManager = buildManager(boostedTargets(runTargets
|
|
540
|
+
let downloadManager = buildManager(boostedTargets(runTargets));
|
|
538
541
|
activeDownloadManager = downloadManager;
|
|
539
542
|
await downloadManager.initialise();
|
|
540
543
|
// Apply initial delay if configured
|
|
@@ -544,102 +547,115 @@ async function createSchedulerRuntime(configPathArg) {
|
|
|
544
547
|
});
|
|
545
548
|
await new Promise((resolve) => setTimeout(resolve, runtimeConfig.initialDelay));
|
|
546
549
|
}
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
slot: slotCtx?.slotId ?? '(ad-hoc)',
|
|
551
|
-
trigger: triggerSource,
|
|
552
|
-
targets: runTargets.map((target) => target.id ?? target.tag ?? target.filterTag ?? target.type),
|
|
553
|
-
});
|
|
554
|
-
logger_1.logger.info('='.repeat(60));
|
|
555
|
-
const startTime = Date.now();
|
|
556
|
-
let allTargetsFailed;
|
|
557
|
-
let releaseLease = () => undefined;
|
|
558
|
-
if (slotCtx && varReleaseLease)
|
|
559
|
-
releaseLease = varReleaseLease;
|
|
550
|
+
// Execution belt (§idle-inflight): this run is live until releaseLease
|
|
551
|
+
// completes. The idle detector must see it even if a durable row lags.
|
|
552
|
+
activeExecutions += 1;
|
|
560
553
|
try {
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
554
|
+
logger_1.logger.info('='.repeat(60));
|
|
555
|
+
logger_1.logger.info('Starting scheduled Pixiv download plan', {
|
|
556
|
+
scheduleId: schedule.id,
|
|
557
|
+
slot: slotCtx?.slotId ?? '(ad-hoc)',
|
|
558
|
+
trigger: triggerSource,
|
|
559
|
+
targets: runTargets.map((target) => target.id ?? target.tag ?? target.filterTag ?? target.type),
|
|
560
|
+
});
|
|
561
|
+
logger_1.logger.info('='.repeat(60));
|
|
562
|
+
const startTime = Date.now();
|
|
563
|
+
let allTargetsFailed;
|
|
564
|
+
let releaseLease = () => undefined;
|
|
565
|
+
if (slotCtx && varReleaseLease)
|
|
566
|
+
releaseLease = varReleaseLease;
|
|
567
|
+
try {
|
|
568
|
+
// Candidate fallback passes (§schedule-recovery): after each pass, cells
|
|
569
|
+
// still mid-fallback (advanced stages) are re-selected with expanded,
|
|
570
|
+
// still-bounded scan limits. Successful sibling cells are NEVER re-run:
|
|
571
|
+
// pendingTargets only returns unconverged cells of this slot.
|
|
572
|
+
for (let pass = 0;; pass += 1) {
|
|
573
|
+
await downloadManager.runAllTargets();
|
|
574
|
+
if (!slotCtx || pass >= maxFallbackStages - 1)
|
|
575
|
+
break;
|
|
576
|
+
const pendingFallback = coordinator
|
|
577
|
+
.pendingTargets(slotCtx.slotId, targets)
|
|
578
|
+
.filter((p) => p.cell &&
|
|
579
|
+
p.cell.fallback_stage > 0 &&
|
|
580
|
+
p.cell.fallback_stage < maxFallbackStages);
|
|
581
|
+
if (pendingFallback.length === 0)
|
|
582
|
+
break;
|
|
583
|
+
downloadManager = buildManager(boostedTargets(pendingFallback.map((p) => p.target)));
|
|
584
|
+
activeDownloadManager = downloadManager;
|
|
585
|
+
await downloadManager.initialise();
|
|
586
|
+
}
|
|
579
587
|
}
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
588
|
+
catch (error) {
|
|
589
|
+
if (!slotCtx || !(error instanceof Error) || !/^All \d+ target\(s\) failed\./.test(error.message)) {
|
|
590
|
+
// Abnormal abort. Two very different situations land here, and treating
|
|
591
|
+
// them as one is what produced the production loop:
|
|
592
|
+
//
|
|
593
|
+
// - this process cancelled itself (scheduler timeout / watchdog) and is
|
|
594
|
+
// still alive. It will never touch the Slot again, so releasing the
|
|
595
|
+
// lease while leaving the status `running` makes the Slot look exactly
|
|
596
|
+
// like a crashed worker: `recoverableSlots()` matches a NULL lease on
|
|
597
|
+
// purpose (its "recorded but never claimed" case), so the recovery
|
|
598
|
+
// sweep re-dispatches the SAME occurrence on every tick, forever.
|
|
599
|
+
// Terminalise it instead: the occurrence is finished, and failed.
|
|
600
|
+
//
|
|
601
|
+
// - the process is going away (`shutdown`). Deliberately leave the Slot
|
|
602
|
+
// non-terminal so recovery resumes the same occurrence after restart.
|
|
603
|
+
if (slotCtx && shouldTerminaliseAbortedSlot(activeAbortOrigin, slotAbandoned)) {
|
|
604
|
+
coordinator.finish(slotCtx, schedule, targets);
|
|
605
|
+
for (const target of targets)
|
|
606
|
+
if (target.id)
|
|
607
|
+
notificationPolicy.noteTerminalRefetchCell(slotCtx.slotId, target.id);
|
|
608
|
+
}
|
|
609
|
+
// Hand the lease back either way: a terminal Slot cannot be re-dispatched,
|
|
610
|
+
// and a non-terminal one (shutdown) must not wait for its TTL to expire.
|
|
611
|
+
releaseLease?.();
|
|
612
|
+
throw error;
|
|
601
613
|
}
|
|
602
|
-
//
|
|
603
|
-
//
|
|
604
|
-
|
|
605
|
-
|
|
614
|
+
// Scheduled Slots treat terminal target failures (failed/no_candidate) as
|
|
615
|
+
// a finished partial/failed aggregate. Finish before returning so the HTTP
|
|
616
|
+
// trigger can report the durable state instead of 500 and never roll up.
|
|
617
|
+
allTargetsFailed = error;
|
|
606
618
|
}
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
619
|
+
finally {
|
|
620
|
+
if (activeDownloadManager === downloadManager)
|
|
621
|
+
activeDownloadManager = null;
|
|
622
|
+
}
|
|
623
|
+
const duration = Math.round((Date.now() - startTime) / 1000);
|
|
624
|
+
if (slotCtx && !slotAbandoned) {
|
|
625
|
+
const summary = coordinator.finish(slotCtx, schedule, targets);
|
|
626
|
+
for (const target of targets)
|
|
627
|
+
if (target.id)
|
|
628
|
+
notificationPolicy.noteTerminalRefetchCell(slotCtx.slotId, target.id);
|
|
629
|
+
// Terminal summaries are for SCHEDULED occurrences (P0-B). Remote manual
|
|
630
|
+
// replacements already report through their own verdict channel
|
|
631
|
+
// (refetchOutcomeUrl); a schedule summary would double-notify the group.
|
|
632
|
+
if (!slotCtx.manualRequestId) {
|
|
633
|
+
notificationPolicy.sendSlotSummary(slotCtx, schedule, summary.cells.map((c) => {
|
|
634
|
+
const t = targets.find((x) => x.id === c.targetId);
|
|
635
|
+
return {
|
|
636
|
+
targetId: c.targetId,
|
|
637
|
+
label: c.targetId,
|
|
638
|
+
workType: t?.type ?? 'unknown',
|
|
639
|
+
status: c.status,
|
|
640
|
+
workId: c.workId,
|
|
641
|
+
error: c.error ?? null,
|
|
642
|
+
};
|
|
643
|
+
}));
|
|
644
|
+
}
|
|
645
|
+
releaseLease();
|
|
646
|
+
}
|
|
647
|
+
if (!slotCtx && allTargetsFailed)
|
|
648
|
+
throw allTargetsFailed;
|
|
649
|
+
logger_1.logger.info('='.repeat(60));
|
|
650
|
+
logger_1.logger.info(`Scheduled download plan finished (took ${duration}s)`, {
|
|
651
|
+
scheduleId: schedule.id,
|
|
652
|
+
slot: slotCtx?.slotId ?? '(ad-hoc)',
|
|
653
|
+
});
|
|
654
|
+
logger_1.logger.info('='.repeat(60));
|
|
611
655
|
}
|
|
612
656
|
finally {
|
|
613
|
-
|
|
614
|
-
activeDownloadManager = null;
|
|
657
|
+
activeExecutions -= 1;
|
|
615
658
|
}
|
|
616
|
-
const duration = Math.round((Date.now() - startTime) / 1000);
|
|
617
|
-
if (slotCtx && !slotAbandoned) {
|
|
618
|
-
const summary = coordinator.finish(slotCtx, schedule, targets);
|
|
619
|
-
for (const target of targets)
|
|
620
|
-
if (target.id)
|
|
621
|
-
notificationPolicy.noteTerminalRefetchCell(slotCtx.slotId, target.id);
|
|
622
|
-
notificationPolicy.sendSlotSummary(slotCtx, schedule, summary.cells.map((c) => {
|
|
623
|
-
const t = targets.find((x) => x.id === c.targetId);
|
|
624
|
-
return {
|
|
625
|
-
targetId: c.targetId,
|
|
626
|
-
label: c.targetId,
|
|
627
|
-
workType: t?.type ?? 'unknown',
|
|
628
|
-
status: c.status,
|
|
629
|
-
workId: c.workId,
|
|
630
|
-
error: c.error ?? null,
|
|
631
|
-
};
|
|
632
|
-
}));
|
|
633
|
-
releaseLease();
|
|
634
|
-
}
|
|
635
|
-
if (!slotCtx && allTargetsFailed)
|
|
636
|
-
throw allTargetsFailed;
|
|
637
|
-
logger_1.logger.info('='.repeat(60));
|
|
638
|
-
logger_1.logger.info(`Scheduled download plan finished (took ${duration}s)`, {
|
|
639
|
-
scheduleId: schedule.id,
|
|
640
|
-
slot: slotCtx?.slotId ?? '(ad-hoc)',
|
|
641
|
-
});
|
|
642
|
-
logger_1.logger.info('='.repeat(60));
|
|
643
659
|
};
|
|
644
660
|
const cancelActive = (reason, origin = 'timeout') => {
|
|
645
661
|
activeAbortOrigin = origin;
|
|
@@ -673,6 +689,7 @@ async function createSchedulerRuntime(configPathArg) {
|
|
|
673
689
|
runJob,
|
|
674
690
|
cancelActive,
|
|
675
691
|
abandonActiveRun,
|
|
692
|
+
activeExecutionCount: () => activeExecutions,
|
|
676
693
|
startOutboxWorker: () => outboxWorker.start(),
|
|
677
694
|
drainOutbox: () => outboxWorker.drainOnce(),
|
|
678
695
|
notifyScheduleFailure: (snapshot, schedule, failure) => notifyScheduleFailure(snapshot, database, schedule, failure),
|
|
@@ -3,6 +3,8 @@ import { OutboxRow } from '../storage/repositories/OutboxRepository';
|
|
|
3
3
|
import { DeliveryDispatcher } from './DeliveryDispatcher';
|
|
4
4
|
import { DeliveryAck } from './DeliveryAck';
|
|
5
5
|
export interface OutboxWorkerOptions {
|
|
6
|
+
/** Reconcile durable notification intents before consuming due rows. */
|
|
7
|
+
beforeDrain?: () => void;
|
|
6
8
|
/** How often to scan for due rows. */
|
|
7
9
|
pollIntervalMs?: number;
|
|
8
10
|
/** Row lease duration while one attempt is in flight. */
|
|
@@ -45,6 +47,7 @@ export declare function backoffDelayMs(attempt: number, base: number, max: numbe
|
|
|
45
47
|
export declare class OutboxWorker {
|
|
46
48
|
private readonly database;
|
|
47
49
|
private readonly dispatcher;
|
|
50
|
+
private readonly options;
|
|
48
51
|
private timer;
|
|
49
52
|
private running;
|
|
50
53
|
private stopped;
|
|
@@ -23,6 +23,7 @@ function backoffDelayMs(attempt, base, max) {
|
|
|
23
23
|
class OutboxWorker {
|
|
24
24
|
database;
|
|
25
25
|
dispatcher;
|
|
26
|
+
options;
|
|
26
27
|
timer = null;
|
|
27
28
|
running = false;
|
|
28
29
|
stopped = false;
|
|
@@ -37,6 +38,7 @@ class OutboxWorker {
|
|
|
37
38
|
constructor(database, dispatcher, options = {}) {
|
|
38
39
|
this.database = database;
|
|
39
40
|
this.dispatcher = dispatcher;
|
|
41
|
+
this.options = options;
|
|
40
42
|
this.pollIntervalMs = options.pollIntervalMs ?? 10_000;
|
|
41
43
|
this.leaseMs = options.leaseMs ?? 120_000;
|
|
42
44
|
this.batchSize = options.batchSize ?? 4;
|
|
@@ -73,6 +75,7 @@ class OutboxWorker {
|
|
|
73
75
|
let retried = 0;
|
|
74
76
|
let dead = 0;
|
|
75
77
|
for (let round = 0; round < maxRounds; round++) {
|
|
78
|
+
this.options.beforeDrain?.();
|
|
76
79
|
const rows = this.database.outbox.claimDue(this.owner, this.leaseMs, this.batchSize);
|
|
77
80
|
if (rows.length === 0)
|
|
78
81
|
break;
|
|
@@ -104,6 +107,7 @@ class OutboxWorker {
|
|
|
104
107
|
return;
|
|
105
108
|
this.running = true;
|
|
106
109
|
try {
|
|
110
|
+
this.options.beforeDrain?.();
|
|
107
111
|
const rows = this.database.outbox.claimDue(this.owner, this.leaseMs, this.batchSize);
|
|
108
112
|
for (const row of rows) {
|
|
109
113
|
if (this.stopped) {
|
|
@@ -51,6 +51,8 @@ export declare class NotificationPolicy {
|
|
|
51
51
|
* use `refetchOutcomeUrl`; generic notifications use `notificationUrl`.
|
|
52
52
|
*/
|
|
53
53
|
private targetsWithUrl;
|
|
54
|
+
/** The durable cells remain the notification intent across crashes and late ACKs. */
|
|
55
|
+
reconcileScheduleSummaries(): void;
|
|
54
56
|
private send;
|
|
55
57
|
/**
|
|
56
58
|
* Report the terminal verdict of a REMOTE MANUAL replacement ("重抓") back to
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.NotificationPolicy = void 0;
|
|
4
4
|
const DeliveryService_1 = require("../delivery/DeliveryService");
|
|
5
|
+
const SlotCoordinator_1 = require("../scheduler/SlotCoordinator");
|
|
5
6
|
const logger_1 = require("../logger");
|
|
6
7
|
/**
|
|
7
8
|
* Central policy for all operational notifications. Handlers/scheduler do not
|
|
@@ -76,8 +77,11 @@ class NotificationPolicy {
|
|
|
76
77
|
}
|
|
77
78
|
/** One consolidated summary per slot, delivered to every notifying target's endpoint. */
|
|
78
79
|
sendSlotSummary(slot, schedule, rows) {
|
|
79
|
-
|
|
80
|
-
|
|
80
|
+
if (slot.manualRequestId || rows.length === 0 || rows.some((r) => !['submitted', 'no_candidate', 'duplicate', 'failed'].includes(r.status)))
|
|
81
|
+
return;
|
|
82
|
+
const memberIds = new Set(rows.map((r) => r.targetId));
|
|
83
|
+
const targets = this.targetsWithUrl('scheduleOutcomeUrl', memberIds);
|
|
84
|
+
if (targets.size === 0)
|
|
81
85
|
return;
|
|
82
86
|
const icon = (s) => s === 'submitted' ? '✅' : s === 'no_candidate' ? '⚠️' : s === 'duplicate' ? '♱' : s === 'delivery_pending' ? '🕓' : '❌';
|
|
83
87
|
const lines = rows.map((r) => `${icon(r.status)} ${r.label}(${r.workType === 'novel' ? '小说' : '插画'})` +
|
|
@@ -93,8 +97,8 @@ class NotificationPolicy {
|
|
|
93
97
|
].join('\n');
|
|
94
98
|
const outcomeStatus = submitted === rows.length ? 'success' : submitted > 0 ? 'partial' : 'failed';
|
|
95
99
|
const service = new DeliveryService_1.DeliveryService(this.database);
|
|
96
|
-
for (const name of targets) {
|
|
97
|
-
service.enqueueNotification(name, text, NotificationPolicy.keys.summary(slot.slotId), undefined, {
|
|
100
|
+
for (const [index, name] of [...targets].entries()) {
|
|
101
|
+
service.enqueueNotification(name, text, NotificationPolicy.keys.summary(slot.slotId) + (index === 0 ? '' : `:${name}`), undefined, {
|
|
98
102
|
scheduleId: schedule.id,
|
|
99
103
|
slotId: slot.slotId,
|
|
100
104
|
status: outcomeStatus,
|
|
@@ -112,9 +116,11 @@ class NotificationPolicy {
|
|
|
112
116
|
* Schedule summaries require `scheduleOutcomeUrl`; manual refetch outcomes
|
|
113
117
|
* use `refetchOutcomeUrl`; generic notifications use `notificationUrl`.
|
|
114
118
|
*/
|
|
115
|
-
targetsWithUrl(urlKey) {
|
|
119
|
+
targetsWithUrl(urlKey, memberIds) {
|
|
116
120
|
const result = new Set();
|
|
117
121
|
for (const target of this.config.targets ?? []) {
|
|
122
|
+
if (memberIds && !memberIds.has(target.id ?? ''))
|
|
123
|
+
continue;
|
|
118
124
|
const deliveryTarget = target.delivery?.target;
|
|
119
125
|
if (!deliveryTarget)
|
|
120
126
|
continue;
|
|
@@ -125,6 +131,26 @@ class NotificationPolicy {
|
|
|
125
131
|
}
|
|
126
132
|
return result;
|
|
127
133
|
}
|
|
134
|
+
/** The durable cells remain the notification intent across crashes and late ACKs. */
|
|
135
|
+
reconcileScheduleSummaries() {
|
|
136
|
+
for (const record of this.database.slots.getUnreportedTerminalSchedules()) {
|
|
137
|
+
const slot = {
|
|
138
|
+
slotId: record.id, scheduleId: record.scheduleId,
|
|
139
|
+
occurrenceAt: record.occurrenceAt ?? 0, occurrenceDate: record.occurrenceDate,
|
|
140
|
+
occurrenceLabel: record.occurrenceLabel, timezone: record.timezone,
|
|
141
|
+
triggerSource: 'http', slotName: record.slotName, slotDate: record.slotDate,
|
|
142
|
+
};
|
|
143
|
+
const schedule = this.config.schedules?.find((item) => item.id === record.scheduleId)
|
|
144
|
+
?? { id: record.scheduleId };
|
|
145
|
+
const targets = this.config.targets.filter((target) => record.targetIds.includes(target.id ?? ''));
|
|
146
|
+
const summary = new SlotCoordinator_1.SlotCoordinator(this.database).finish(slot, schedule, targets);
|
|
147
|
+
this.sendSlotSummary(slot, schedule, summary.cells.map((cell) => ({
|
|
148
|
+
...cell, label: cell.targetId,
|
|
149
|
+
workType: targets.find((target) => target.id === cell.targetId)?.type ?? 'unknown',
|
|
150
|
+
error: cell.error ?? null,
|
|
151
|
+
})));
|
|
152
|
+
}
|
|
153
|
+
}
|
|
128
154
|
send(targetName, key, text) {
|
|
129
155
|
try {
|
|
130
156
|
new DeliveryService_1.DeliveryService(this.database).enqueueNotification(targetName, text, key);
|
package/dist/package.json
CHANGED
|
@@ -89,6 +89,8 @@ export declare class SlotRepository extends BaseRepository {
|
|
|
89
89
|
/** Target ids materialized for a slot (stable membership; falls back to []). */
|
|
90
90
|
getSlotTargetIds(id: string): string[];
|
|
91
91
|
getRecentSlots(limit?: number): SlotRecord[];
|
|
92
|
+
/** Completed cells whose summary enqueue or aggregate rollup was interrupted. */
|
|
93
|
+
getUnreportedTerminalSchedules(): SlotRecord[];
|
|
92
94
|
markSlotStatus(id: string, status: SlotStatus, error?: string): void;
|
|
93
95
|
/** All cells for a slot (one per target). */
|
|
94
96
|
getCells(slotId: string): SlotItemRecord[];
|
|
@@ -71,6 +71,18 @@ class SlotRepository extends BaseRepository_1.BaseRepository {
|
|
|
71
71
|
.all(limit);
|
|
72
72
|
return rows.map((r) => this.toSlot(r));
|
|
73
73
|
}
|
|
74
|
+
/** Completed cells whose summary enqueue or aggregate rollup was interrupted. */
|
|
75
|
+
getUnreportedTerminalSchedules() {
|
|
76
|
+
const rows = this.db.prepare(`SELECT s.* FROM schedule_slots s
|
|
77
|
+
WHERE s.manual_request_id IS NULL AND s.status != 'expired'
|
|
78
|
+
AND EXISTS (SELECT 1 FROM schedule_slot_items c WHERE c.slot_id = s.id)
|
|
79
|
+
AND NOT EXISTS (SELECT 1 FROM schedule_slot_items c WHERE c.slot_id = s.id
|
|
80
|
+
AND c.status NOT IN ('submitted', 'no_candidate', 'duplicate', 'failed'))
|
|
81
|
+
AND (s.status IN ('pending', 'running') OR NOT EXISTS (
|
|
82
|
+
SELECT 1 FROM outbox o WHERE o.idempotency_key = 'notification:' || s.id || ':summary'
|
|
83
|
+
))`).all();
|
|
84
|
+
return rows.map((row) => this.toSlot(row));
|
|
85
|
+
}
|
|
74
86
|
markSlotStatus(id, status, error) {
|
|
75
87
|
const stamp = status === 'running' ? 'started_at' : status === 'success' || status === 'partial' || status === 'failed' ? 'completed_at' : null;
|
|
76
88
|
const sets = ['status = @status', 'last_error = @error'];
|
package/dist/version.js
CHANGED
|
@@ -2,5 +2,5 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.BUILD = void 0;
|
|
4
4
|
// GENERATED by scripts/write-version.js — do not edit manually.
|
|
5
|
-
exports.BUILD = { version: '2.20.
|
|
5
|
+
exports.BUILD = { version: '2.20.5', commit: '9352c0253571' };
|
|
6
6
|
//# sourceMappingURL=version.js.map
|
package/dist/webui/package.json
CHANGED
package/package.json
CHANGED