pixivflow 2.20.4 → 2.20.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +7 -3
- package/README.md +6 -5
- package/dist/commands/SchedulerCommand.js +3 -0
- package/dist/commands/SchedulerIdleLifecycle.d.ts +9 -0
- package/dist/commands/SchedulerIdleLifecycle.js +2 -1
- package/dist/commands/scheduler-runtime.d.ts +8 -0
- package/dist/commands/scheduler-runtime.js +104 -94
- package/dist/package.json +1 -1
- package/dist/version.js +1 -1
- package/dist/webui/package.json +1 -1
- package/package.json +1 -1
package/README.en.md
CHANGED
|
@@ -22,7 +22,9 @@ pixivflow --help
|
|
|
22
22
|
```
|
|
23
23
|
|
|
24
24
|
For servers, prefer the Docker Compose setup described in
|
|
25
|
-
[DOCKER.md](docs/DOCKER.md). To
|
|
25
|
+
[DOCKER.md](docs/DOCKER.md). To compose PixivFlow with TelePost, use the optional
|
|
26
|
+
[pixivflow-telepost-deploy](https://github.com/redtidev1918/pixivflow-telepost-deploy)
|
|
27
|
+
deployment and operations kit. To build from source:
|
|
26
28
|
|
|
27
29
|
```bash
|
|
28
30
|
git clone https://github.com/redtidev1918/PixivFlow.git
|
|
@@ -92,8 +94,10 @@ conflict.
|
|
|
92
94
|
|
|
93
95
|
Each target can use `storageMode: "persistent"` (the default, keep files) or
|
|
94
96
|
`storageMode: "cache"` (send files to a named delivery target and delete them
|
|
95
|
-
only after success). The delivery layer is service-independent;
|
|
96
|
-
|
|
97
|
+
only after success). The delivery layer is service-independent;
|
|
98
|
+
[TelePost](https://github.com/redtidev1918/TelePost) and
|
|
99
|
+
[telepress](https://github.com/redtidev1918/telepress) are example downstreams,
|
|
100
|
+
not dependencies. This example merely translates an HTTP multipart submission API into configuration:
|
|
97
101
|
|
|
98
102
|
```json
|
|
99
103
|
{
|
package/README.md
CHANGED
|
@@ -22,10 +22,9 @@ npm install -g pixivflow
|
|
|
22
22
|
pixivflow --help
|
|
23
23
|
```
|
|
24
24
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
—— 一套配置,支持国内/海外、有/无公网 IP、VPS/Fly.io 任意场景。
|
|
25
|
+
服务器部署见 [DOCKER.md](docs/DOCKER.md)。需要把 PixivFlow 与 TelePost 组合运行时,再使用
|
|
26
|
+
[pixivflow-telepost-deploy](https://github.com/redtidev1918/pixivflow-telepost-deploy);
|
|
27
|
+
它是可选的部署与运维套件。
|
|
29
28
|
从源码构建:
|
|
30
29
|
|
|
31
30
|
```bash
|
|
@@ -111,7 +110,9 @@ pixivflow scheduler # 按 cron 配置长期挂机自动收集
|
|
|
111
110
|
- **`cache`**:下载后投给一个「交付目标」(比如投稿机器人),对方确认收到后才删本地文件,省磁盘。
|
|
112
111
|
|
|
113
112
|
「交付目标」就是一段配置:告诉 PixivFlow 把文件 POST 到哪个地址、带哪些字段。
|
|
114
|
-
|
|
113
|
+
它不绑定具体服务,可指向任意兼容的 HTTP 接口;
|
|
114
|
+
[TelePost](https://github.com/redtidev1918/TelePost) 与
|
|
115
|
+
[telepress](https://github.com/redtidev1918/telepress) 只是示例下游。示例:
|
|
115
116
|
|
|
116
117
|
```json
|
|
117
118
|
{
|
|
@@ -255,6 +255,9 @@ class SchedulerCommand extends Command_1.BaseCommand {
|
|
|
255
255
|
// attempts are exhausted, which does not block exit), so this
|
|
256
256
|
// cannot hold the machine open indefinitely.
|
|
257
257
|
pendingOutbox: outbox.pending,
|
|
258
|
+
// Second belt: never exit underneath an in-flight slot execution
|
|
259
|
+
// even if a durable row ever looks a beat early (§idle-inflight).
|
|
260
|
+
activeExecutions: runtime.activeExecutionCount(),
|
|
258
261
|
};
|
|
259
262
|
},
|
|
260
263
|
idleGraceMs: scheduling.idleGraceMs,
|
|
@@ -41,6 +41,15 @@ export interface IdleSnapshot {
|
|
|
41
41
|
processingOutbox: number;
|
|
42
42
|
/** Undelivered outbox rows (pending, or waiting on a bounded retry backoff). */
|
|
43
43
|
pendingOutbox: number;
|
|
44
|
+
/**
|
|
45
|
+
* In-process slot executions currently awaiting async work (candidate scan,
|
|
46
|
+
* download, delivery). This is a SECOND belt behind the durable ledger: even
|
|
47
|
+
* if a DB row is an instant ahead of its awaiting Promise (or a mis-write
|
|
48
|
+
* ever terminalized a row early), the idle detector must not exit underneath
|
|
49
|
+
* a live execution. In-memory only — a process crash still resumes from the
|
|
50
|
+
* durable slot/outbox rows.
|
|
51
|
+
*/
|
|
52
|
+
activeExecutions: number;
|
|
44
53
|
}
|
|
45
54
|
/** True only when the worker has no work of any kind left. */
|
|
46
55
|
export declare function isIdle(snapshot: IdleSnapshot): boolean;
|
|
@@ -38,7 +38,8 @@ const logger_1 = require("../logger");
|
|
|
38
38
|
function isIdle(snapshot) {
|
|
39
39
|
return (snapshot.activeSlots === 0 &&
|
|
40
40
|
snapshot.processingOutbox === 0 &&
|
|
41
|
-
snapshot.pendingOutbox === 0
|
|
41
|
+
snapshot.pendingOutbox === 0 &&
|
|
42
|
+
snapshot.activeExecutions === 0);
|
|
42
43
|
}
|
|
43
44
|
/** Default idle grace: long enough to merge two schedules ~10 minutes apart. */
|
|
44
45
|
exports.DEFAULT_IDLE_GRACE_MS = 10 * 60 * 1000;
|
|
@@ -100,6 +100,14 @@ export interface SchedulerRuntime {
|
|
|
100
100
|
notifyScheduleFailure(snapshot: StandaloneConfig, schedule: ScheduleConfig, failure: JobFailure): Promise<void>;
|
|
101
101
|
/** Start the independent outbox pump (long-running daemon). */
|
|
102
102
|
startOutboxWorker(): void;
|
|
103
|
+
/**
|
|
104
|
+
* Number of slot executions currently awaiting async work in THIS process
|
|
105
|
+
* (candidate scan / download / delivery). Bounded second belt for the idle
|
|
106
|
+
* lifecycle: exit is only allowed when this is 0 AND the durable ledger is
|
|
107
|
+
* empty (§idle-inflight). In-memory only; crash recovery still rides the
|
|
108
|
+
* durable slot/outbox rows.
|
|
109
|
+
*/
|
|
110
|
+
activeExecutionCount(): number;
|
|
103
111
|
/** Drain due outbox rows once (run-once / watchdog wake). */
|
|
104
112
|
drainOutbox(): Promise<{
|
|
105
113
|
processed: number;
|
|
@@ -275,6 +275,7 @@ async function createSchedulerRuntime(configPathArg) {
|
|
|
275
275
|
notificationPolicy.noteTerminalRefetchCell(delivery.slotId, delivery.targetId);
|
|
276
276
|
},
|
|
277
277
|
});
|
|
278
|
+
let activeExecutions = 0;
|
|
278
279
|
const runJob = async (snapshot, schedule, options = {}) => {
|
|
279
280
|
const { onlyTarget, adhoc = false, slot: providedSlot } = options;
|
|
280
281
|
// A scheduled run is cron by default; an HTTP/manual trigger passes its own.
|
|
@@ -546,107 +547,115 @@ async function createSchedulerRuntime(configPathArg) {
|
|
|
546
547
|
});
|
|
547
548
|
await new Promise((resolve) => setTimeout(resolve, runtimeConfig.initialDelay));
|
|
548
549
|
}
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
slot: slotCtx?.slotId ?? '(ad-hoc)',
|
|
553
|
-
trigger: triggerSource,
|
|
554
|
-
targets: runTargets.map((target) => target.id ?? target.tag ?? target.filterTag ?? target.type),
|
|
555
|
-
});
|
|
556
|
-
logger_1.logger.info('='.repeat(60));
|
|
557
|
-
const startTime = Date.now();
|
|
558
|
-
let allTargetsFailed;
|
|
559
|
-
let releaseLease = () => undefined;
|
|
560
|
-
if (slotCtx && varReleaseLease)
|
|
561
|
-
releaseLease = varReleaseLease;
|
|
550
|
+
// Execution belt (§idle-inflight): this run is live until releaseLease
|
|
551
|
+
// completes. The idle detector must see it even if a durable row lags.
|
|
552
|
+
activeExecutions += 1;
|
|
562
553
|
try {
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
554
|
+
logger_1.logger.info('='.repeat(60));
|
|
555
|
+
logger_1.logger.info('Starting scheduled Pixiv download plan', {
|
|
556
|
+
scheduleId: schedule.id,
|
|
557
|
+
slot: slotCtx?.slotId ?? '(ad-hoc)',
|
|
558
|
+
trigger: triggerSource,
|
|
559
|
+
targets: runTargets.map((target) => target.id ?? target.tag ?? target.filterTag ?? target.type),
|
|
560
|
+
});
|
|
561
|
+
logger_1.logger.info('='.repeat(60));
|
|
562
|
+
const startTime = Date.now();
|
|
563
|
+
let allTargetsFailed;
|
|
564
|
+
let releaseLease = () => undefined;
|
|
565
|
+
if (slotCtx && varReleaseLease)
|
|
566
|
+
releaseLease = varReleaseLease;
|
|
567
|
+
try {
|
|
568
|
+
// Candidate fallback passes (§schedule-recovery): after each pass, cells
|
|
569
|
+
// still mid-fallback (advanced stages) are re-selected with expanded,
|
|
570
|
+
// still-bounded scan limits. Successful sibling cells are NEVER re-run:
|
|
571
|
+
// pendingTargets only returns unconverged cells of this slot.
|
|
572
|
+
for (let pass = 0;; pass += 1) {
|
|
573
|
+
await downloadManager.runAllTargets();
|
|
574
|
+
if (!slotCtx || pass >= maxFallbackStages - 1)
|
|
575
|
+
break;
|
|
576
|
+
const pendingFallback = coordinator
|
|
577
|
+
.pendingTargets(slotCtx.slotId, targets)
|
|
578
|
+
.filter((p) => p.cell &&
|
|
579
|
+
p.cell.fallback_stage > 0 &&
|
|
580
|
+
p.cell.fallback_stage < maxFallbackStages);
|
|
581
|
+
if (pendingFallback.length === 0)
|
|
582
|
+
break;
|
|
583
|
+
downloadManager = buildManager(boostedTargets(pendingFallback.map((p) => p.target)));
|
|
584
|
+
activeDownloadManager = downloadManager;
|
|
585
|
+
await downloadManager.initialise();
|
|
586
|
+
}
|
|
581
587
|
}
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
588
|
+
catch (error) {
|
|
589
|
+
if (!slotCtx || !(error instanceof Error) || !/^All \d+ target\(s\) failed\./.test(error.message)) {
|
|
590
|
+
// Abnormal abort. Two very different situations land here, and treating
|
|
591
|
+
// them as one is what produced the production loop:
|
|
592
|
+
//
|
|
593
|
+
// - this process cancelled itself (scheduler timeout / watchdog) and is
|
|
594
|
+
// still alive. It will never touch the Slot again, so releasing the
|
|
595
|
+
// lease while leaving the status `running` makes the Slot look exactly
|
|
596
|
+
// like a crashed worker: `recoverableSlots()` matches a NULL lease on
|
|
597
|
+
// purpose (its "recorded but never claimed" case), so the recovery
|
|
598
|
+
// sweep re-dispatches the SAME occurrence on every tick, forever.
|
|
599
|
+
// Terminalise it instead: the occurrence is finished, and failed.
|
|
600
|
+
//
|
|
601
|
+
// - the process is going away (`shutdown`). Deliberately leave the Slot
|
|
602
|
+
// non-terminal so recovery resumes the same occurrence after restart.
|
|
603
|
+
if (slotCtx && shouldTerminaliseAbortedSlot(activeAbortOrigin, slotAbandoned)) {
|
|
604
|
+
coordinator.finish(slotCtx, schedule, targets);
|
|
605
|
+
for (const target of targets)
|
|
606
|
+
if (target.id)
|
|
607
|
+
notificationPolicy.noteTerminalRefetchCell(slotCtx.slotId, target.id);
|
|
608
|
+
}
|
|
609
|
+
// Hand the lease back either way: a terminal Slot cannot be re-dispatched,
|
|
610
|
+
// and a non-terminal one (shutdown) must not wait for its TTL to expire.
|
|
611
|
+
releaseLease?.();
|
|
612
|
+
throw error;
|
|
613
|
+
}
|
|
614
|
+
// Scheduled Slots treat terminal target failures (failed/no_candidate) as
|
|
615
|
+
// a finished partial/failed aggregate. Finish before returning so the HTTP
|
|
616
|
+
// trigger can report the durable state instead of 500 and never roll up.
|
|
617
|
+
allTargetsFailed = error;
|
|
618
|
+
}
|
|
619
|
+
finally {
|
|
620
|
+
if (activeDownloadManager === downloadManager)
|
|
621
|
+
activeDownloadManager = null;
|
|
622
|
+
}
|
|
623
|
+
const duration = Math.round((Date.now() - startTime) / 1000);
|
|
624
|
+
if (slotCtx && !slotAbandoned) {
|
|
625
|
+
const summary = coordinator.finish(slotCtx, schedule, targets);
|
|
626
|
+
for (const target of targets)
|
|
627
|
+
if (target.id)
|
|
628
|
+
notificationPolicy.noteTerminalRefetchCell(slotCtx.slotId, target.id);
|
|
629
|
+
// Terminal summaries are for SCHEDULED occurrences (P0-B). Remote manual
|
|
630
|
+
// replacements already report through their own verdict channel
|
|
631
|
+
// (refetchOutcomeUrl); a schedule summary would double-notify the group.
|
|
632
|
+
if (!slotCtx.manualRequestId) {
|
|
633
|
+
notificationPolicy.sendSlotSummary(slotCtx, schedule, summary.cells.map((c) => {
|
|
634
|
+
const t = targets.find((x) => x.id === c.targetId);
|
|
635
|
+
return {
|
|
636
|
+
targetId: c.targetId,
|
|
637
|
+
label: c.targetId,
|
|
638
|
+
workType: t?.type ?? 'unknown',
|
|
639
|
+
status: c.status,
|
|
640
|
+
workId: c.workId,
|
|
641
|
+
error: c.error ?? null,
|
|
642
|
+
};
|
|
643
|
+
}));
|
|
603
644
|
}
|
|
604
|
-
|
|
605
|
-
// and a non-terminal one (shutdown) must not wait for its TTL to expire.
|
|
606
|
-
releaseLease?.();
|
|
607
|
-
throw error;
|
|
645
|
+
releaseLease();
|
|
608
646
|
}
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
647
|
+
if (!slotCtx && allTargetsFailed)
|
|
648
|
+
throw allTargetsFailed;
|
|
649
|
+
logger_1.logger.info('='.repeat(60));
|
|
650
|
+
logger_1.logger.info(`Scheduled download plan finished (took ${duration}s)`, {
|
|
651
|
+
scheduleId: schedule.id,
|
|
652
|
+
slot: slotCtx?.slotId ?? '(ad-hoc)',
|
|
653
|
+
});
|
|
654
|
+
logger_1.logger.info('='.repeat(60));
|
|
613
655
|
}
|
|
614
656
|
finally {
|
|
615
|
-
|
|
616
|
-
activeDownloadManager = null;
|
|
617
|
-
}
|
|
618
|
-
const duration = Math.round((Date.now() - startTime) / 1000);
|
|
619
|
-
if (slotCtx && !slotAbandoned) {
|
|
620
|
-
const summary = coordinator.finish(slotCtx, schedule, targets);
|
|
621
|
-
for (const target of targets)
|
|
622
|
-
if (target.id)
|
|
623
|
-
notificationPolicy.noteTerminalRefetchCell(slotCtx.slotId, target.id);
|
|
624
|
-
// Terminal summaries are for SCHEDULED occurrences (P0-B). Remote manual
|
|
625
|
-
// replacements already report through their own verdict channel
|
|
626
|
-
// (refetchOutcomeUrl); a schedule summary would double-notify the group.
|
|
627
|
-
if (!slotCtx.manualRequestId) {
|
|
628
|
-
notificationPolicy.sendSlotSummary(slotCtx, schedule, summary.cells.map((c) => {
|
|
629
|
-
const t = targets.find((x) => x.id === c.targetId);
|
|
630
|
-
return {
|
|
631
|
-
targetId: c.targetId,
|
|
632
|
-
label: c.targetId,
|
|
633
|
-
workType: t?.type ?? 'unknown',
|
|
634
|
-
status: c.status,
|
|
635
|
-
workId: c.workId,
|
|
636
|
-
error: c.error ?? null,
|
|
637
|
-
};
|
|
638
|
-
}));
|
|
639
|
-
}
|
|
640
|
-
releaseLease();
|
|
657
|
+
activeExecutions -= 1;
|
|
641
658
|
}
|
|
642
|
-
if (!slotCtx && allTargetsFailed)
|
|
643
|
-
throw allTargetsFailed;
|
|
644
|
-
logger_1.logger.info('='.repeat(60));
|
|
645
|
-
logger_1.logger.info(`Scheduled download plan finished (took ${duration}s)`, {
|
|
646
|
-
scheduleId: schedule.id,
|
|
647
|
-
slot: slotCtx?.slotId ?? '(ad-hoc)',
|
|
648
|
-
});
|
|
649
|
-
logger_1.logger.info('='.repeat(60));
|
|
650
659
|
};
|
|
651
660
|
const cancelActive = (reason, origin = 'timeout') => {
|
|
652
661
|
activeAbortOrigin = origin;
|
|
@@ -680,6 +689,7 @@ async function createSchedulerRuntime(configPathArg) {
|
|
|
680
689
|
runJob,
|
|
681
690
|
cancelActive,
|
|
682
691
|
abandonActiveRun,
|
|
692
|
+
activeExecutionCount: () => activeExecutions,
|
|
683
693
|
startOutboxWorker: () => outboxWorker.start(),
|
|
684
694
|
drainOutbox: () => outboxWorker.drainOnce(),
|
|
685
695
|
notifyScheduleFailure: (snapshot, schedule, failure) => notifyScheduleFailure(snapshot, database, schedule, failure),
|
package/dist/package.json
CHANGED
package/dist/version.js
CHANGED
|
@@ -2,5 +2,5 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.BUILD = void 0;
|
|
4
4
|
// GENERATED by scripts/write-version.js — do not edit manually.
|
|
5
|
-
exports.BUILD = { version: '2.20.
|
|
5
|
+
exports.BUILD = { version: '2.20.5', commit: '9352c0253571' };
|
|
6
6
|
//# sourceMappingURL=version.js.map
|
package/dist/webui/package.json
CHANGED
package/package.json
CHANGED