pixivflow 3.4.0 → 3.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/SchedulerCommand.js +8 -0
- package/dist/commands/scheduler-runtime.d.ts +22 -1
- package/dist/commands/scheduler-runtime.js +24 -0
- package/dist/config/types.d.ts +10 -1
- package/dist/config/validation.js +5 -0
- package/dist/delivery/DeliveryService.d.ts +7 -0
- package/dist/delivery/HttpMultipartDelivery.js +5 -0
- package/dist/delivery/types.d.ts +6 -0
- package/dist/domain/media/NovelCoverPolicy.d.ts +25 -3
- package/dist/domain/media/NovelCoverPolicy.js +24 -8
- package/dist/download/DownloadManager.js +1 -0
- package/dist/download/NovelDownloader.d.ts +5 -2
- package/dist/download/NovelDownloader.js +24 -5
- package/dist/notification/NotificationPolicy.js +28 -2
- package/dist/package.json +1 -1
- package/dist/scheduler/ManualJobService.d.ts +10 -0
- package/dist/scheduler/ManualJobService.js +6 -1
- package/dist/scheduler/SlotCoordinator.js +7 -0
- package/dist/version.js +1 -1
- package/dist/webui/package.json +1 -1
- package/package.json +1 -1
|
@@ -12,6 +12,7 @@ const ScheduleTriggerServer_1 = require("../scheduler/ScheduleTriggerServer");
|
|
|
12
12
|
const SlotCoordinator_1 = require("../scheduler/SlotCoordinator");
|
|
13
13
|
const ManualJobAdmission_1 = require("../scheduler/ManualJobAdmission");
|
|
14
14
|
const ManualJobService_1 = require("../scheduler/ManualJobService");
|
|
15
|
+
const JobCancellation_1 = require("../scheduler/JobCancellation");
|
|
15
16
|
const ManualRefetchAdapter_1 = require("../scheduler/ManualRefetchAdapter");
|
|
16
17
|
const schedules_1 = require("../scheduler/schedules");
|
|
17
18
|
const targetRoutes_1 = require("../delivery/targetRoutes");
|
|
@@ -100,6 +101,13 @@ class SchedulerCommand extends Command_1.BaseCommand {
|
|
|
100
101
|
database: runtime.database,
|
|
101
102
|
config: resolveConfig,
|
|
102
103
|
admission,
|
|
104
|
+
// A consumer cancel must bite the WORK, not only the ledger: the ledger
|
|
105
|
+
// write already happened inside `cancelJob`, so this only stops the
|
|
106
|
+
// in-flight download loop (and the `Scheduler.running` it holds, which
|
|
107
|
+
// otherwise refuses every later admission for the same schedule).
|
|
108
|
+
onCancel: (slotId) => {
|
|
109
|
+
runtime.cancelSlot(slotId, JobCancellation_1.CANCELLED_BY_CONSUMER_MESSAGE);
|
|
110
|
+
},
|
|
103
111
|
});
|
|
104
112
|
// Durable dispatch. The trigger endpoint records the occurrence FIRST,
|
|
105
113
|
// then hands it to the shared scheduler and answers immediately. It must
|
|
@@ -62,8 +62,14 @@ export declare function withDeliveryMode<T extends {
|
|
|
62
62
|
* - `shutdown`: the process is going away. The Slot is deliberately left
|
|
63
63
|
* non-terminal so recovery resumes the same occurrence after restart; no
|
|
64
64
|
* worker survives to duplicate it.
|
|
65
|
+
* - `consumer`: an operator/consumer cancelled THIS job through the protocol
|
|
66
|
+
* `POST /jobs/{job_id}/cancel`. The ledger already took the Slot terminal in
|
|
67
|
+
* one transaction (`cancelConsumerJob`, reason `cancelled_by_consumer`), so
|
|
68
|
+
* the abort path must not roll it up a second time — doing so would overwrite
|
|
69
|
+
* the cancel verdict with a generic failure and flip the protocol status from
|
|
70
|
+
* `cancelled` back to `failed`.
|
|
65
71
|
*/
|
|
66
|
-
export type CancelOrigin = 'timeout' | 'shutdown';
|
|
72
|
+
export type CancelOrigin = 'timeout' | 'shutdown' | 'consumer';
|
|
67
73
|
/**
|
|
68
74
|
* Decide a scheduled run's Slot fate when `runAllTargets()` aborts abnormally
|
|
69
75
|
* instead of reporting per-target failures.
|
|
@@ -90,6 +96,21 @@ export interface SchedulerRuntime {
|
|
|
90
96
|
* (`shutdown`: leave the Slot non-terminal so recovery resumes it).
|
|
91
97
|
*/
|
|
92
98
|
cancelActive(reason: string, origin?: CancelOrigin): void;
|
|
99
|
+
/**
|
|
100
|
+
* Cancel the in-flight run **only if it is the one owning `slotId`**.
|
|
101
|
+
*
|
|
102
|
+
* This is what makes a protocol consumer cancel bite: `cancelConsumerJob`
|
|
103
|
+
* terminalises the ledger in one transaction, but the work itself is an
|
|
104
|
+
* in-process download loop that knows nothing about the ledger. Without this
|
|
105
|
+
* call the cancelled job keeps consuming until it finishes on its own, and the
|
|
106
|
+
* holding `Scheduler.running` keeps refusing every later admission for that
|
|
107
|
+
* schedule (`scheduler_busy`) for the whole remaining run.
|
|
108
|
+
*
|
|
109
|
+
* Returns true when this call cancelled that Slot's live run. A slot another
|
|
110
|
+
* run owns (or a job that is only queued) is untouched — its ledger terminal is
|
|
111
|
+
* all there is to do.
|
|
112
|
+
*/
|
|
113
|
+
cancelSlot(slotId: string, reason: string): boolean;
|
|
93
114
|
/**
|
|
94
115
|
* The active run never settled after its timeout and drain window. Stop
|
|
95
116
|
* renewing its lease and take its Slot terminal so recovery cannot re-dispatch
|
|
@@ -98,6 +98,11 @@ function shouldTerminaliseAbortedSlot(origin, slotAbandoned) {
|
|
|
98
98
|
// only overwrite it when the wedged job finally settles.
|
|
99
99
|
if (slotAbandoned)
|
|
100
100
|
return false;
|
|
101
|
+
// A consumer cancel is already terminal in the ledger (`cancelled_by_consumer`,
|
|
102
|
+
// written by cancelConsumerJob in one transaction). Finishing it again would
|
|
103
|
+
// replace that verdict with a generic failure.
|
|
104
|
+
if (origin === 'consumer')
|
|
105
|
+
return false;
|
|
101
106
|
// Shutdown is not a failure: recovery is meant to resume this occurrence.
|
|
102
107
|
return origin !== 'shutdown';
|
|
103
108
|
}
|
|
@@ -291,6 +296,12 @@ async function createSchedulerRuntime(configPathArg) {
|
|
|
291
296
|
* `null` while no cancellation has been requested for the current run.
|
|
292
297
|
*/
|
|
293
298
|
let activeAbortOrigin = null;
|
|
299
|
+
/**
|
|
300
|
+
* The Slot the in-flight run owns, if any. Set where the lease is claimed and
|
|
301
|
+
* cleared where it is released, so `cancelSlot()` can tell "this job is the one
|
|
302
|
+
* running right now" from "this job is merely queued behind another run".
|
|
303
|
+
*/
|
|
304
|
+
let activeSlotId = null;
|
|
294
305
|
// Independently-pumped durable outbox (content + notifications). Started in
|
|
295
306
|
// the long-running scheduler daemon; run-once drains explicitly before exit.
|
|
296
307
|
const deliveryDispatcher = new DeliveryDispatcher_1.DeliveryDispatcher(config.delivery, buildProxyUrl(config.network));
|
|
@@ -452,6 +463,7 @@ async function createSchedulerRuntime(configPathArg) {
|
|
|
452
463
|
// This run owns the Slot now: any cancellation recorded against a previous
|
|
453
464
|
// run must not decide how THIS run's abort is handled.
|
|
454
465
|
activeAbortOrigin = null;
|
|
466
|
+
activeSlotId = activeSlot.slotId;
|
|
455
467
|
let cancelled = false;
|
|
456
468
|
const heartbeat = setInterval(() => {
|
|
457
469
|
// A cancelled/timed-out run must stop renewing its lease. An infinitely
|
|
@@ -468,6 +480,7 @@ async function createSchedulerRuntime(configPathArg) {
|
|
|
468
480
|
varReleaseLease = () => {
|
|
469
481
|
clearInterval(heartbeat);
|
|
470
482
|
activeLeaseHooks = null;
|
|
483
|
+
activeSlotId = null;
|
|
471
484
|
coordinator.releaseRunLease(activeSlot.slotId, runOwner);
|
|
472
485
|
};
|
|
473
486
|
activeLeaseHooks = {
|
|
@@ -497,6 +510,7 @@ async function createSchedulerRuntime(configPathArg) {
|
|
|
497
510
|
database.slots.markSlotStatus(activeSlot.slotId, 'failed', `abandoned after scheduler timeout; no delivery (${reason})`);
|
|
498
511
|
coordinator.releaseRunLease(activeSlot.slotId, runOwner);
|
|
499
512
|
activeLeaseHooks = null;
|
|
513
|
+
activeSlotId = null;
|
|
500
514
|
},
|
|
501
515
|
};
|
|
502
516
|
}
|
|
@@ -778,6 +792,15 @@ async function createSchedulerRuntime(configPathArg) {
|
|
|
778
792
|
// or left recoverable is decided by `origin` in the abort path of runJob.
|
|
779
793
|
activeLeaseHooks?.stopHeartbeat();
|
|
780
794
|
};
|
|
795
|
+
const cancelSlot = (slotId, reason) => {
|
|
796
|
+
// A cancellation is only meaningful for the run that owns THIS Slot. Every
|
|
797
|
+
// other case (queued behind another run, or already terminal) has nothing to
|
|
798
|
+
// abort: the durable ledger write is the whole effect.
|
|
799
|
+
if (!activeSlotId || activeSlotId !== slotId)
|
|
800
|
+
return false;
|
|
801
|
+
cancelActive(reason, 'consumer');
|
|
802
|
+
return true;
|
|
803
|
+
};
|
|
781
804
|
const abandonActiveRun = (reason) => {
|
|
782
805
|
// Cancellation did not take effect inside the drain window — a request that
|
|
783
806
|
// ignores the abort. This run will never settle, so finish the lease
|
|
@@ -800,6 +823,7 @@ async function createSchedulerRuntime(configPathArg) {
|
|
|
800
823
|
tokenMaintenance,
|
|
801
824
|
runJob,
|
|
802
825
|
cancelActive,
|
|
826
|
+
cancelSlot,
|
|
803
827
|
abandonActiveRun,
|
|
804
828
|
activeExecutionCount: () => activeExecutions,
|
|
805
829
|
startOutboxWorker: () => outboxWorker.start(),
|
package/dist/config/types.d.ts
CHANGED
|
@@ -945,10 +945,19 @@ export interface StandaloneConfig {
|
|
|
945
945
|
* classified. 'skip' (default) is safe mode: never ship an unclassifiable
|
|
946
946
|
* cover, and log `coverType=unknown` so a future Pixiv format change is
|
|
947
947
|
* visible instead of silently leaking designs again. 'keep' prefers
|
|
948
|
-
* availability over certainty.
|
|
948
|
+
* availability over certainty.
|
|
949
|
+
*
|
|
950
|
+
* `probeFailed` governs the separate case where the cover probe itself
|
|
951
|
+
* failed (network / auth / rate limit) and the bytes were never seen.
|
|
952
|
+
* 'skip' (default) is safe mode for the same reason: most novel covers ARE
|
|
953
|
+
* generated designs, so keeping an unseen cover re-ships what the 640x900
|
|
954
|
+
* classifier exists to suppress; the cost is one illustration when a real
|
|
955
|
+
* cover is transiently unreachable. 'keep' restores availability-first.
|
|
956
|
+
* Either way the `coverType=probe_failed` warning stays visible.
|
|
949
957
|
*/
|
|
950
958
|
novelCover?: {
|
|
951
959
|
unknown?: 'skip' | 'keep';
|
|
960
|
+
probeFailed?: 'skip' | 'keep';
|
|
952
961
|
};
|
|
953
962
|
};
|
|
954
963
|
}
|
|
@@ -715,6 +715,11 @@ function validateConfig(config, location, databasePath) {
|
|
|
715
715
|
config.download.novelCover.unknown !== 'keep') {
|
|
716
716
|
errors.push(`download.novelCover.unknown: Must be "skip" or "keep" (got ${String(config.download.novelCover.unknown)})`);
|
|
717
717
|
}
|
|
718
|
+
if (config.download.novelCover?.probeFailed !== undefined &&
|
|
719
|
+
config.download.novelCover.probeFailed !== 'skip' &&
|
|
720
|
+
config.download.novelCover.probeFailed !== 'keep') {
|
|
721
|
+
errors.push(`download.novelCover.probeFailed: Must be "skip" or "keep" (got ${String(config.download.novelCover.probeFailed)})`);
|
|
722
|
+
}
|
|
718
723
|
}
|
|
719
724
|
// The bounded candidate scan is what stops a page full of duplicates from
|
|
720
725
|
// burning a whole scheduled slot, so a non-integer value is reported here.
|
|
@@ -47,6 +47,13 @@ export interface RefetchOutcomePayload {
|
|
|
47
47
|
requestId: string;
|
|
48
48
|
/** 'no_alternative' | 'failed' (replacement success rides the submission). */
|
|
49
49
|
disposition: 'no_alternative' | 'failed';
|
|
50
|
+
/**
|
|
51
|
+
* The cross-boundary, closed-vocabulary protocol error code (protocol/v1
|
|
52
|
+
* `$defs/Error.code`). The consumer writes it into its `failure_code` CODE
|
|
53
|
+
* column, so raw upstream text must never appear here.
|
|
54
|
+
*/
|
|
55
|
+
reasonCode?: string;
|
|
56
|
+
/** Bounded, single-line, human-readable business message (never a raw body). */
|
|
50
57
|
reason?: string;
|
|
51
58
|
workId?: string;
|
|
52
59
|
/** Bounded candidate-scan bookkeeping for diagnostics (spec-compatible). */
|
|
@@ -173,6 +173,11 @@ class HttpMultipartDelivery {
|
|
|
173
173
|
? {
|
|
174
174
|
request_id: outcome.requestId,
|
|
175
175
|
disposition: outcome.disposition,
|
|
176
|
+
// The closed-vocabulary code goes to TelePost's `failure_code` CODE
|
|
177
|
+
// column; `reason` is only the bounded human business message it
|
|
178
|
+
// sanitizes into `terminal_reason`. Raw upstream text (an nginx 502
|
|
179
|
+
// HTML page) must never appear on the wire.
|
|
180
|
+
reason_code: outcome.reasonCode,
|
|
176
181
|
reason: outcome.reason,
|
|
177
182
|
work_id: outcome.workId,
|
|
178
183
|
scanned: outcome.scanned,
|
package/dist/delivery/types.d.ts
CHANGED
|
@@ -154,6 +154,12 @@ export interface DeliveryNotificationRequest {
|
|
|
154
154
|
refetchOutcome?: {
|
|
155
155
|
requestId: string;
|
|
156
156
|
disposition: 'no_alternative' | 'failed';
|
|
157
|
+
/**
|
|
158
|
+
* The cross-boundary, closed-vocabulary protocol error code (protocol/v1
|
|
159
|
+
* `$defs/Error.code`); raw upstream text never crosses the boundary.
|
|
160
|
+
*/
|
|
161
|
+
reasonCode?: string;
|
|
162
|
+
/** Bounded, single-line, human-readable business message (never a raw body). */
|
|
157
163
|
reason?: string;
|
|
158
164
|
workId?: string;
|
|
159
165
|
scanned?: number;
|
|
@@ -15,14 +15,36 @@ export interface NovelCoverPolicy {
|
|
|
15
15
|
* 'keep': prefer availability over certainty.
|
|
16
16
|
*/
|
|
17
17
|
unknownCover: 'skip' | 'keep';
|
|
18
|
+
/**
|
|
19
|
+
* What to do when the cover probe itself failed (network / auth / rate
|
|
20
|
+
* limit), so the bytes were never fetched and no content type exists.
|
|
21
|
+
*
|
|
22
|
+
* 'skip' (default): safe mode — most novel covers ARE Pixiv's generated
|
|
23
|
+
* designs, so keeping an unseen probe failure re-ships exactly the covers
|
|
24
|
+
* the 640x900 classifier exists to suppress. The cost is one illustration
|
|
25
|
+
* when a real author cover happens to be transiently unreachable.
|
|
26
|
+
* 'keep': prefer availability over certainty — a failed probe never costs a
|
|
27
|
+
* cover, at the price of re-shipping generated designs whenever the CDN or
|
|
28
|
+
* the account is unreachable.
|
|
29
|
+
*/
|
|
30
|
+
probeFailed: 'skip' | 'keep';
|
|
18
31
|
}
|
|
19
|
-
/**
|
|
32
|
+
/**
|
|
33
|
+
* Production-safe default: an unclassifiable cover is never shipped, and a
|
|
34
|
+
* probe that never produced bytes is not treated as evidence that the cover is
|
|
35
|
+
* real content.
|
|
36
|
+
*/
|
|
20
37
|
export declare const DEFAULT_NOVEL_COVER_POLICY: NovelCoverPolicy;
|
|
21
38
|
/**
|
|
22
39
|
* Classify fetched cover bytes. Returns `unknown` when the image header cannot
|
|
23
40
|
* be read (no decoder in this project, so only the header is inspected).
|
|
24
41
|
*/
|
|
25
42
|
export declare function classifyNovelCover(cover: ArrayBuffer | Uint8Array | null | undefined): NovelCoverType;
|
|
26
|
-
/**
|
|
27
|
-
|
|
43
|
+
/**
|
|
44
|
+
* Everything `coverDeliveryDecision` can be asked about: the three results
|
|
45
|
+
* `classifyNovelCover` returns, plus the probe that never produced bytes.
|
|
46
|
+
*/
|
|
47
|
+
export type NovelCoverOutcome = NovelCoverType | 'probe_failed';
|
|
48
|
+
/** The delivery decision for one cover outcome under a policy. */
|
|
49
|
+
export declare function coverDeliveryDecision(policy: NovelCoverPolicy, outcome: NovelCoverOutcome): 'deliver' | 'skip';
|
|
28
50
|
//# sourceMappingURL=NovelCoverPolicy.d.ts.map
|
|
@@ -31,8 +31,15 @@ exports.coverDeliveryDecision = coverDeliveryDecision;
|
|
|
31
31
|
* shipping generated covers again.
|
|
32
32
|
*
|
|
33
33
|
* A FAILED probe (network / auth / rate limit) is deliberately NOT `unknown`:
|
|
34
|
-
* it is `probe_failed`
|
|
35
|
-
*
|
|
34
|
+
* it is `probe_failed` — the bytes were never seen, so no content type exists.
|
|
35
|
+
* It is a policy decision like `unknown` (`download.novelCover.probeFailed`),
|
|
36
|
+
* and the production-safe default is `skip`: the majority of novel covers ARE
|
|
37
|
+
* Pixiv's generated designs, so a probe failure that keeps the cover re-ships
|
|
38
|
+
* exactly the payloads the 640x900 classifier exists to suppress. `keep` stays
|
|
39
|
+
* available for an operator who prefers availability over certainty (a
|
|
40
|
+
* transient fetch error must never cost a real cover). Either way the loud
|
|
41
|
+
* `coverType=probe_failed` log line stays, so the decision is observable
|
|
42
|
+
* instead of silent.
|
|
36
43
|
*/
|
|
37
44
|
const imageDimensions_1 = require("../../utils/imageDimensions");
|
|
38
45
|
/**
|
|
@@ -43,8 +50,15 @@ const imageDimensions_1 = require("../../utils/imageDimensions");
|
|
|
43
50
|
*/
|
|
44
51
|
exports.PIXIV_GENERATED_COVER_WIDTH = 640;
|
|
45
52
|
exports.PIXIV_GENERATED_COVER_HEIGHT = 900;
|
|
46
|
-
/**
|
|
47
|
-
|
|
53
|
+
/**
|
|
54
|
+
* Production-safe default: an unclassifiable cover is never shipped, and a
|
|
55
|
+
* probe that never produced bytes is not treated as evidence that the cover is
|
|
56
|
+
* real content.
|
|
57
|
+
*/
|
|
58
|
+
exports.DEFAULT_NOVEL_COVER_POLICY = {
|
|
59
|
+
unknownCover: 'skip',
|
|
60
|
+
probeFailed: 'skip',
|
|
61
|
+
};
|
|
48
62
|
/**
|
|
49
63
|
* Classify fetched cover bytes. Returns `unknown` when the image header cannot
|
|
50
64
|
* be read (no decoder in this project, so only the header is inspected).
|
|
@@ -59,12 +73,14 @@ function classifyNovelCover(cover) {
|
|
|
59
73
|
}
|
|
60
74
|
return 'custom';
|
|
61
75
|
}
|
|
62
|
-
/** The delivery decision for one
|
|
63
|
-
function coverDeliveryDecision(policy,
|
|
64
|
-
if (
|
|
76
|
+
/** The delivery decision for one cover outcome under a policy. */
|
|
77
|
+
function coverDeliveryDecision(policy, outcome) {
|
|
78
|
+
if (outcome === 'pixiv_generated')
|
|
65
79
|
return 'skip';
|
|
66
|
-
if (
|
|
80
|
+
if (outcome === 'unknown')
|
|
67
81
|
return policy.unknownCover === 'keep' ? 'deliver' : 'skip';
|
|
82
|
+
if (outcome === 'probe_failed')
|
|
83
|
+
return policy.probeFailed === 'keep' ? 'deliver' : 'skip';
|
|
68
84
|
return 'deliver';
|
|
69
85
|
}
|
|
70
86
|
//# sourceMappingURL=NovelCoverPolicy.js.map
|
|
@@ -132,6 +132,7 @@ class DownloadManager {
|
|
|
132
132
|
: MaterializationPolicy_1.DEFAULT_MATERIALIZATION_POLICY;
|
|
133
133
|
const novelCoverPolicy = {
|
|
134
134
|
unknownCover: config.download?.novelCover?.unknown ?? NovelCoverPolicy_1.DEFAULT_NOVEL_COVER_POLICY.unknownCover,
|
|
135
|
+
probeFailed: config.download?.novelCover?.probeFailed ?? NovelCoverPolicy_1.DEFAULT_NOVEL_COVER_POLICY.probeFailed,
|
|
135
136
|
};
|
|
136
137
|
this.novelDownloader = new NovelDownloader_1.NovelDownloader(client, database, fileService, database, undefined, materialization, novelCoverPolicy);
|
|
137
138
|
this.planner = new DownloadPlanner_1.DownloadPlanner(database, {
|
|
@@ -34,8 +34,11 @@ export declare class NovelDownloader {
|
|
|
34
34
|
* - unknown → policy decision, default 'skip' (safe mode), so a future
|
|
35
35
|
* Pixiv cover-format change surfaces as a loud
|
|
36
36
|
* `coverType=unknown` log instead of leaking silently
|
|
37
|
-
*
|
|
38
|
-
*
|
|
37
|
+
* - probe_failed → policy decision, default 'skip': the bytes were never
|
|
38
|
+
* seen, and most novel covers are generated designs, so
|
|
39
|
+
* keeping an unseen cover re-ships what this classifier
|
|
40
|
+
* exists to suppress. `download.novelCover.probeFailed:
|
|
41
|
+
* keep` restores the availability-first behaviour.
|
|
39
42
|
*/
|
|
40
43
|
private resolveCoverUrl;
|
|
41
44
|
}
|
|
@@ -418,8 +418,11 @@ class NovelDownloader {
|
|
|
418
418
|
* - unknown → policy decision, default 'skip' (safe mode), so a future
|
|
419
419
|
* Pixiv cover-format change surfaces as a loud
|
|
420
420
|
* `coverType=unknown` log instead of leaking silently
|
|
421
|
-
*
|
|
422
|
-
*
|
|
421
|
+
* - probe_failed → policy decision, default 'skip': the bytes were never
|
|
422
|
+
* seen, and most novel covers are generated designs, so
|
|
423
|
+
* keeping an unseen cover re-ships what this classifier
|
|
424
|
+
* exists to suppress. `download.novelCover.probeFailed:
|
|
425
|
+
* keep` restores the availability-first behaviour.
|
|
423
426
|
*/
|
|
424
427
|
async resolveCoverUrl(novelId, coverUrl) {
|
|
425
428
|
const normalized = (0, novelCover_1.normalizeNovelCoverUrl)(coverUrl);
|
|
@@ -430,13 +433,29 @@ class NovelDownloader {
|
|
|
430
433
|
cover = await this.client.downloadImage(normalized);
|
|
431
434
|
}
|
|
432
435
|
catch (error) {
|
|
433
|
-
|
|
436
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
437
|
+
// A failed probe is a policy decision, not an automatic keep: most novel
|
|
438
|
+
// covers ARE Pixiv's generated designs, so silently shipping an unseen
|
|
439
|
+
// cover re-ships exactly what the 640x900 classifier exists to suppress.
|
|
440
|
+
const probeDecision = (0, NovelCoverPolicy_1.coverDeliveryDecision)(this.novelCoverPolicy, 'probe_failed');
|
|
441
|
+
if (probeDecision === 'deliver') {
|
|
442
|
+
logger_1.logger.warn(`Novel ${novelId} cover probe failed; keeping the cover per novelCover.probeFailed=keep (coverType=probe_failed)`, {
|
|
443
|
+
novelId,
|
|
444
|
+
coverUrl: normalized,
|
|
445
|
+
coverType: 'probe_failed',
|
|
446
|
+
policy: this.novelCoverPolicy.probeFailed,
|
|
447
|
+
reason,
|
|
448
|
+
});
|
|
449
|
+
return normalized;
|
|
450
|
+
}
|
|
451
|
+
logger_1.logger.warn(`Novel ${novelId} cover probe failed; skipping the cover per novelCover.probeFailed=skip (coverType=probe_failed)`, {
|
|
434
452
|
novelId,
|
|
435
453
|
coverUrl: normalized,
|
|
436
454
|
coverType: 'probe_failed',
|
|
437
|
-
|
|
455
|
+
policy: this.novelCoverPolicy.probeFailed,
|
|
456
|
+
reason,
|
|
438
457
|
});
|
|
439
|
-
return
|
|
458
|
+
return null;
|
|
440
459
|
}
|
|
441
460
|
const coverType = (0, NovelCoverPolicy_1.classifyNovelCover)(cover);
|
|
442
461
|
const decision = (0, NovelCoverPolicy_1.coverDeliveryDecision)(this.novelCoverPolicy, coverType);
|
|
@@ -4,6 +4,7 @@ exports.NotificationPolicy = void 0;
|
|
|
4
4
|
const DeliveryService_1 = require("../delivery/DeliveryService");
|
|
5
5
|
const targetRoutes_1 = require("../delivery/targetRoutes");
|
|
6
6
|
const TargetOutcome_1 = require("../scheduler/TargetOutcome");
|
|
7
|
+
const ProtocolErrors_1 = require("../scheduler/ProtocolErrors");
|
|
7
8
|
const SlotCoordinator_1 = require("../scheduler/SlotCoordinator");
|
|
8
9
|
const logger_1 = require("../logger");
|
|
9
10
|
/**
|
|
@@ -227,6 +228,20 @@ class NotificationPolicy {
|
|
|
227
228
|
const name = this.targetName(target);
|
|
228
229
|
if (!name)
|
|
229
230
|
return;
|
|
231
|
+
// A consumer/operator cancel is NOT a refetch failure (finding D). The
|
|
232
|
+
// cancelled cell keeps its terminal `cancelled_by_consumer` verdict, and no
|
|
233
|
+
// `disposition:"failed"` outcome may ever be emitted for it — the operator
|
|
234
|
+
// deliberately stopped the work, and the cancellation is already surfaced
|
|
235
|
+
// through the job status / events channel, never as a failed refetch report.
|
|
236
|
+
const cancelledCell = target.id ? this.database.slots.getCell(slot.slotId, target.id) : null;
|
|
237
|
+
if (cancelledCell?.terminalReasonCode === ProtocolErrors_1.CANCELLED_BY_CONSUMER) {
|
|
238
|
+
logger_1.logger.info('Refetch outcome suppressed: job was cancelled by the consumer', {
|
|
239
|
+
slot: slot.slotId,
|
|
240
|
+
target: target.id,
|
|
241
|
+
requestId,
|
|
242
|
+
});
|
|
243
|
+
return;
|
|
244
|
+
}
|
|
230
245
|
const deliveryTarget = this.config.delivery?.targets?.[name];
|
|
231
246
|
if (!deliveryTarget || deliveryTarget.type !== 'httpMultipart')
|
|
232
247
|
return;
|
|
@@ -239,12 +254,22 @@ class NotificationPolicy {
|
|
|
239
254
|
});
|
|
240
255
|
return;
|
|
241
256
|
}
|
|
257
|
+
// Translate the internal outcome ONCE, through the existing classifier:
|
|
258
|
+
// the wire carries the closed-vocabulary protocol code plus the short
|
|
259
|
+
// Chinese business message. The raw technical text (`outcome.error` /
|
|
260
|
+
// `outcome.reason`) never crosses the boundary — a live acceptance run
|
|
261
|
+
// shipped a 220-char multi-line nginx 502 HTML page into TelePost's
|
|
262
|
+
// `failure_code` CODE column that way.
|
|
263
|
+
const normalized = (0, TargetOutcome_1.terminalReasonFor)(outcome);
|
|
264
|
+
const reasonCode = (0, TargetOutcome_1.protocolErrorCodeForTerminalReason)(normalized?.code) ?? undefined;
|
|
265
|
+
const reason = (normalized?.message ?? '').trim() || undefined;
|
|
242
266
|
let payload;
|
|
243
267
|
if (outcome.kind === 'no_candidate' || outcome.kind === 'duplicate') {
|
|
244
268
|
payload = {
|
|
245
269
|
requestId,
|
|
246
270
|
disposition: 'no_alternative',
|
|
247
|
-
|
|
271
|
+
reasonCode,
|
|
272
|
+
reason,
|
|
248
273
|
workId: outcome.kind === 'duplicate' ? outcome.workId : undefined,
|
|
249
274
|
...scanCounts(outcome.scan),
|
|
250
275
|
};
|
|
@@ -253,7 +278,8 @@ class NotificationPolicy {
|
|
|
253
278
|
payload = {
|
|
254
279
|
requestId,
|
|
255
280
|
disposition: 'failed',
|
|
256
|
-
|
|
281
|
+
reasonCode,
|
|
282
|
+
reason,
|
|
257
283
|
...scanCounts(outcome.scan),
|
|
258
284
|
};
|
|
259
285
|
}
|
package/dist/package.json
CHANGED
|
@@ -25,6 +25,16 @@ export interface ManualJobServiceDeps {
|
|
|
25
25
|
config(): StandaloneConfig;
|
|
26
26
|
/** The shared admission path — the same instance the legacy shim uses. */
|
|
27
27
|
admission: ManualJobAdmission;
|
|
28
|
+
/**
|
|
29
|
+
* Ask the runtime to abort the in-flight run when the cancelled job is the one
|
|
30
|
+
* executing right now.
|
|
31
|
+
*
|
|
32
|
+
* The ledger write is the source of truth and happens first; this only stops
|
|
33
|
+
* the in-process download loop from running to completion after it has been
|
|
34
|
+
* cancelled. A job that is merely queued behind another run has no live run to
|
|
35
|
+
* abort, and the hook is then never called.
|
|
36
|
+
*/
|
|
37
|
+
onCancel?(slotId: string): void;
|
|
28
38
|
/** Injected clock, for tests. */
|
|
29
39
|
now?(): number;
|
|
30
40
|
}
|
|
@@ -78,10 +78,15 @@ class ManualJobService {
|
|
|
78
78
|
* that terminalises the work and stops further deliveries.
|
|
79
79
|
*/
|
|
80
80
|
cancelJob(jobId) {
|
|
81
|
-
(0, JobCancellation_1.cancelConsumerJob)(this.deps.database, jobId, this.now());
|
|
81
|
+
const result = (0, JobCancellation_1.cancelConsumerJob)(this.deps.database, jobId, this.now());
|
|
82
82
|
// The cancellation is now durable, so its terminal event must be too — a
|
|
83
83
|
// consumer that polls after cancelling must never see an unterminated stream.
|
|
84
84
|
this.events.reconcile(jobId);
|
|
85
|
+
// Only a cancel that actually transitioned the ledger has a live run worth
|
|
86
|
+
// aborting. The ledger write came first, so the abort path cannot resurrect
|
|
87
|
+
// the job; it only makes the running download loop stop consuming.
|
|
88
|
+
if (result.cancelled)
|
|
89
|
+
this.deps.onCancel?.(jobId);
|
|
85
90
|
return this.viewBySlotId(jobId);
|
|
86
91
|
}
|
|
87
92
|
viewBySlotId(slotId) {
|
|
@@ -245,6 +245,13 @@ class SlotCoordinator {
|
|
|
245
245
|
const cell = this.database.slots.getCell(slotId, targetId);
|
|
246
246
|
if (!cell)
|
|
247
247
|
return;
|
|
248
|
+
// A consumer/operator cancel is FINAL (see JobCancellation.ts). The abort of
|
|
249
|
+
// the interrupted run still lands here through the normal per-target outcome
|
|
250
|
+
// path; a late `failed` must never re-verdict a cancelled cell, or the
|
|
251
|
+
// protocol status flips from `cancelled` back to `failed` ~10s after the
|
|
252
|
+
// operator's cancel (finding D). Cancel verdicts are never re-written.
|
|
253
|
+
if (cell.terminalReasonCode === ProtocolErrors_1.CANCELLED_BY_CONSUMER)
|
|
254
|
+
return;
|
|
248
255
|
switch (outcome.kind) {
|
|
249
256
|
case 'submitted':
|
|
250
257
|
this.database.slots.lockCellWork(slotId, targetId, outcome.workId, outcome.workType);
|
package/dist/version.js
CHANGED
|
@@ -2,5 +2,5 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.BUILD = void 0;
|
|
4
4
|
// GENERATED by scripts/write-version.js — do not edit manually.
|
|
5
|
-
exports.BUILD = { version: '3.4.
|
|
5
|
+
exports.BUILD = { version: '3.4.2', commit: '4db0bf21a785' };
|
|
6
6
|
//# sourceMappingURL=version.js.map
|
package/dist/webui/package.json
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pixivflow",
|
|
3
|
-
"version": "3.4.
|
|
3
|
+
"version": "3.4.2",
|
|
4
4
|
"description": "🎨 Pixiv 下载、筛选与自动收集工具 - 批量下载插画和小说、按标签/热度/日期筛选、定时任务与可靠 HTTP 交付 | Pixiv downloader and automation toolkit with filtering, scheduling and reliable HTTP delivery",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|