pixivflow 2.28.1 → 2.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/scheduler-runtime.js +1 -0
- package/dist/delivery/DeliveryService.d.ts +1 -0
- package/dist/delivery/HttpMultipartDelivery.js +1 -0
- package/dist/delivery/types.d.ts +2 -0
- package/dist/download/handlers/IllustrationTargetHandler.d.ts +5 -0
- package/dist/download/handlers/IllustrationTargetHandler.js +25 -3
- package/dist/download/handlers/NovelTargetHandler.d.ts +4 -0
- package/dist/download/handlers/NovelTargetHandler.js +21 -3
- package/dist/notification/NotificationPolicy.d.ts +1 -0
- package/dist/notification/NotificationPolicy.js +1 -0
- package/dist/package.json +1 -1
- package/dist/scheduler/SlotCoordinator.d.ts +6 -0
- package/dist/scheduler/SlotCoordinator.js +26 -0
- package/dist/scheduler/TargetOutcome.d.ts +46 -0
- package/dist/scheduler/TargetOutcome.js +60 -0
- package/dist/storage/DatabaseMigration.js +4 -0
- package/dist/storage/repositories/SlotRepository.d.ts +11 -0
- package/dist/storage/repositories/SlotRepository.js +17 -0
- package/dist/topic/TopicPipeline.d.ts +2 -0
- package/dist/topic/TopicPipeline.js +5 -1
- package/dist/version.js +1 -1
- package/dist/webui/package.json +1 -1
- package/package.json +1 -1
|
@@ -176,6 +176,7 @@ class HttpMultipartDelivery {
|
|
|
176
176
|
stage: t.stage ?? null,
|
|
177
177
|
retryable: t.retryable ?? null,
|
|
178
178
|
operator_hint: t.operator_hint ?? null,
|
|
179
|
+
candidate_report: t.candidate_report ?? null,
|
|
179
180
|
})),
|
|
180
181
|
}
|
|
181
182
|
: { text: request.text, idempotency_key: request.idempotencyKey };
|
package/dist/delivery/types.d.ts
CHANGED
|
@@ -146,6 +146,8 @@ export interface DeliveryNotificationRequest {
|
|
|
146
146
|
/** Whether a later/manual attempt may succeed; terminal does not mean auto-retry pending. */
|
|
147
147
|
retryable?: boolean | null;
|
|
148
148
|
operator_hint?: string | null;
|
|
149
|
+
/** Phase 1 Candidate Supply Report ({fetched, selected, rejected, reasons}). */
|
|
150
|
+
candidate_report?: Record<string, unknown> | null;
|
|
149
151
|
}>;
|
|
150
152
|
};
|
|
151
153
|
}
|
|
@@ -25,6 +25,11 @@ export declare class IllustrationTargetHandler {
|
|
|
25
25
|
* explicit verdict instead of an ambiguous "completed".
|
|
26
26
|
*/
|
|
27
27
|
private scan;
|
|
28
|
+
/**
|
|
29
|
+
* Upstream topic-supply funnel accumulated across a lookback scan (Phase 1
|
|
30
|
+
* Candidate Report: fetched/selected/rejected + reasons).
|
|
31
|
+
*/
|
|
32
|
+
private supplyRep;
|
|
28
33
|
/**
|
|
29
34
|
* Global candidate-scan bound (`download.candidateScanLimit`), supplied by
|
|
30
35
|
* DownloadManager. This is what lets the FETCH stage ask for more than one
|
|
@@ -29,6 +29,11 @@ class IllustrationTargetHandler {
|
|
|
29
29
|
* explicit verdict instead of an ambiguous "completed".
|
|
30
30
|
*/
|
|
31
31
|
scan = null;
|
|
32
|
+
/**
|
|
33
|
+
* Upstream topic-supply funnel accumulated across a lookback scan (Phase 1
|
|
34
|
+
* Candidate Report: fetched/selected/rejected + reasons).
|
|
35
|
+
*/
|
|
36
|
+
supplyRep = null;
|
|
32
37
|
/**
|
|
33
38
|
* Global candidate-scan bound (`download.candidateScanLimit`), supplied by
|
|
34
39
|
* DownloadManager. This is what lets the FETCH stage ask for more than one
|
|
@@ -68,6 +73,7 @@ class IllustrationTargetHandler {
|
|
|
68
73
|
async handle(target, execution) {
|
|
69
74
|
this.outcomes = [];
|
|
70
75
|
this.scan = null;
|
|
76
|
+
this.supplyRep = null;
|
|
71
77
|
this.execution = execution && (0, WorkIdentity_1.isSingleWorkCell)(target) ? execution : null;
|
|
72
78
|
// A cell that already owns a work is in RECOVERY, not in a new selection.
|
|
73
79
|
// Crash/shutdown recovery is not an intentional second run: running the
|
|
@@ -113,6 +119,9 @@ class IllustrationTargetHandler {
|
|
|
113
119
|
* scheduled slot report success after submitting nothing.
|
|
114
120
|
*/
|
|
115
121
|
summarize(target) {
|
|
122
|
+
if (this.scan && this.supplyRep) {
|
|
123
|
+
this.scan = { ...this.scan, supply: (0, TargetOutcome_1.withScanSkips)(this.supplyRep, this.scan) };
|
|
124
|
+
}
|
|
116
125
|
const scan = this.scan ?? undefined;
|
|
117
126
|
const submitted = this.outcomes.find((o) => o.kind === 'submitted');
|
|
118
127
|
if (submitted)
|
|
@@ -198,7 +207,8 @@ class IllustrationTargetHandler {
|
|
|
198
207
|
}
|
|
199
208
|
async fetchIllustrations(target, mode) {
|
|
200
209
|
if (mode === 'topic') {
|
|
201
|
-
|
|
210
|
+
const { works } = await this.fetchTopicIllustrations(target);
|
|
211
|
+
return works;
|
|
202
212
|
}
|
|
203
213
|
if (mode === 'ranking') {
|
|
204
214
|
return this.fetchRankingIllustrations(target);
|
|
@@ -223,7 +233,19 @@ class IllustrationTargetHandler {
|
|
|
223
233
|
const pipeline = this.topicPipelineFactory();
|
|
224
234
|
const { works, selection } = await pipeline.selectWorks(target, 'illustration', day, selectionLimit, target.topicDiscovery ?? {}, target.candidateCollection ?? {});
|
|
225
235
|
logger_1.logger.info(`Topic "${topic}" illustration: tags=${selection.resolvedTagCount} raw=${selection.rawCount} deduped=${selection.dedupedCount} aiExcluded=${selection.aiExcludedCount} accepted=${selection.acceptedCount} candidates=${works.length} target=${limit}`);
|
|
226
|
-
|
|
236
|
+
// Candidate Supply Observability: fold this lookback day's upstream funnel
|
|
237
|
+
// into the target's accumulated Candidate Report.
|
|
238
|
+
const report = (0, TargetOutcome_1.emptyCandidateSupplyReport)();
|
|
239
|
+
report.fetched = selection.rawCount;
|
|
240
|
+
report.selected = selection.acceptedCount;
|
|
241
|
+
report.rejected = Math.max(0, selection.rawCount - selection.acceptedCount);
|
|
242
|
+
report.reasons = [
|
|
243
|
+
{ code: 'duplicate', count: selection.duplicateRemovedCount },
|
|
244
|
+
{ code: 'ai_filtered', count: selection.aiExcludedCount },
|
|
245
|
+
{ code: 'metadata_filtered', count: Math.max(0, selection.dedupedCount - selection.acceptedCount) },
|
|
246
|
+
].filter((r) => r.count > 0);
|
|
247
|
+
this.supplyRep = (0, TargetOutcome_1.mergeCandidateSupplyReports)(this.supplyRep ?? undefined, report);
|
|
248
|
+
return { works, selection };
|
|
227
249
|
}
|
|
228
250
|
async handleTopicWithLookback(target, displayTag) {
|
|
229
251
|
const requested = target.limit || 1;
|
|
@@ -241,7 +263,7 @@ class IllustrationTargetHandler {
|
|
|
241
263
|
fallbackOffset: offset,
|
|
242
264
|
});
|
|
243
265
|
}
|
|
244
|
-
const illusts = await this.fetchTopicIllustrations(attemptTarget);
|
|
266
|
+
const { works: illusts } = await this.fetchTopicIllustrations(attemptTarget);
|
|
245
267
|
const result = await this.pipeline.run(illusts, attemptTarget, 'illustration', (illust, tag) => this.downloadAndDeliver(illust, tag, attemptTarget));
|
|
246
268
|
// The lookback loop is one bounded scan: accumulate every day's
|
|
247
269
|
// skips/outages so the final verdict covers all candidates attempted.
|
|
@@ -24,6 +24,10 @@ export declare class NovelTargetHandler {
|
|
|
24
24
|
* explicit verdict instead of an ambiguous "completed".
|
|
25
25
|
*/
|
|
26
26
|
private scan;
|
|
27
|
+
/**
|
|
28
|
+
* Upstream topic-supply funnel accumulated across a lookback scan.
|
|
29
|
+
*/
|
|
30
|
+
private supplyRep;
|
|
27
31
|
/**
|
|
28
32
|
* Global candidate-scan bound (`download.candidateScanLimit`), supplied by
|
|
29
33
|
* DownloadManager. This is what lets the FETCH stage ask for more than one
|
|
@@ -29,6 +29,10 @@ class NovelTargetHandler {
|
|
|
29
29
|
* explicit verdict instead of an ambiguous "completed".
|
|
30
30
|
*/
|
|
31
31
|
scan = null;
|
|
32
|
+
/**
|
|
33
|
+
* Upstream topic-supply funnel accumulated across a lookback scan.
|
|
34
|
+
*/
|
|
35
|
+
supplyRep = null;
|
|
32
36
|
/**
|
|
33
37
|
* Global candidate-scan bound (`download.candidateScanLimit`), supplied by
|
|
34
38
|
* DownloadManager. This is what lets the FETCH stage ask for more than one
|
|
@@ -68,6 +72,7 @@ class NovelTargetHandler {
|
|
|
68
72
|
async handle(target, execution) {
|
|
69
73
|
this.outcomes = [];
|
|
70
74
|
this.scan = null;
|
|
75
|
+
this.supplyRep = null;
|
|
71
76
|
this.execution = execution && (0, WorkIdentity_1.isSingleWorkCell)(target) ? execution : null;
|
|
72
77
|
// A cell that already owns a work is in RECOVERY, not in a new selection.
|
|
73
78
|
// Crash/shutdown recovery is not an intentional second run: running the
|
|
@@ -126,6 +131,9 @@ class NovelTargetHandler {
|
|
|
126
131
|
* single-work RECOVERY path, whose cell identity is fixed.
|
|
127
132
|
*/
|
|
128
133
|
summarize() {
|
|
134
|
+
if (this.scan && this.supplyRep) {
|
|
135
|
+
this.scan = { ...this.scan, supply: (0, TargetOutcome_1.withScanSkips)(this.supplyRep, this.scan) };
|
|
136
|
+
}
|
|
129
137
|
const scan = this.scan ?? undefined;
|
|
130
138
|
const submitted = this.outcomes.find((o) => o.kind === 'submitted');
|
|
131
139
|
if (submitted)
|
|
@@ -200,7 +208,8 @@ class NovelTargetHandler {
|
|
|
200
208
|
}
|
|
201
209
|
async fetchNovels(target, mode) {
|
|
202
210
|
if (mode === 'topic') {
|
|
203
|
-
|
|
211
|
+
const { works } = await this.fetchTopicNovels(target);
|
|
212
|
+
return works;
|
|
204
213
|
}
|
|
205
214
|
if (mode === 'ranking') {
|
|
206
215
|
return this.fetchRankingNovels(target);
|
|
@@ -223,7 +232,16 @@ class NovelTargetHandler {
|
|
|
223
232
|
const pipeline = this.topicPipelineFactory();
|
|
224
233
|
const { works, selection } = await pipeline.selectWorks(target, 'novel', day, selectionLimit, target.topicDiscovery ?? {}, target.candidateCollection ?? {});
|
|
225
234
|
logger_1.logger.info(`Topic "${topic}" novel: tags=${selection.resolvedTagCount} raw=${selection.rawCount} deduped=${selection.dedupedCount} accepted=${selection.acceptedCount} candidates=${works.length} target=${limit}`);
|
|
226
|
-
|
|
235
|
+
const report = (0, TargetOutcome_1.emptyCandidateSupplyReport)();
|
|
236
|
+
report.fetched = selection.rawCount;
|
|
237
|
+
report.selected = selection.acceptedCount;
|
|
238
|
+
report.rejected = Math.max(0, selection.rawCount - selection.acceptedCount);
|
|
239
|
+
report.reasons = [
|
|
240
|
+
{ code: 'duplicate', count: selection.duplicateRemovedCount },
|
|
241
|
+
{ code: 'metadata_filtered', count: Math.max(0, selection.dedupedCount - selection.acceptedCount) },
|
|
242
|
+
].filter((r) => r.count > 0);
|
|
243
|
+
this.supplyRep = (0, TargetOutcome_1.mergeCandidateSupplyReports)(this.supplyRep ?? undefined, report);
|
|
244
|
+
return { works, selection };
|
|
227
245
|
}
|
|
228
246
|
async handleTopicWithLookback(target, displayTag) {
|
|
229
247
|
const requested = target.limit || 1;
|
|
@@ -253,7 +271,7 @@ class NovelTargetHandler {
|
|
|
253
271
|
fallbackOffset: offset,
|
|
254
272
|
});
|
|
255
273
|
}
|
|
256
|
-
const novels = await this.fetchTopicNovels(attemptTarget);
|
|
274
|
+
const { works: novels } = await this.fetchTopicNovels(attemptTarget);
|
|
257
275
|
totalFound += novels.length;
|
|
258
276
|
const result = await this.pipeline.run(novels, attemptTarget, 'novel', (novel, tag) => this.downloadAndDeliver(novel, tag, attemptTarget));
|
|
259
277
|
// The lookback loop is ONE bounded scan: accumulate every day's
|
|
@@ -46,6 +46,7 @@ export declare class NotificationPolicy {
|
|
|
46
46
|
error: string | null;
|
|
47
47
|
terminal_reason_code?: string | null;
|
|
48
48
|
reason?: string | null;
|
|
49
|
+
candidateReport?: Record<string, unknown> | null;
|
|
49
50
|
}>): void;
|
|
50
51
|
/**
|
|
51
52
|
* Delivery targets whose HTTP target declares the given outcome URL.
|
package/dist/package.json
CHANGED
|
@@ -67,6 +67,8 @@ export interface SlotCellSummary {
|
|
|
67
67
|
terminalReasonCode?: string | null;
|
|
68
68
|
/** User-facing business reason message (§terminal-reason). */
|
|
69
69
|
terminalReasonMessage?: string | null;
|
|
70
|
+
/** Phase 1 Candidate Supply Report attached to the durable cell. */
|
|
71
|
+
candidateReport?: Record<string, unknown> | null;
|
|
70
72
|
}
|
|
71
73
|
export interface SlotRunSummary {
|
|
72
74
|
scheduleId: string;
|
|
@@ -82,6 +84,8 @@ export interface ScheduleOutcomeTarget {
|
|
|
82
84
|
status: CellStatus;
|
|
83
85
|
work_id: string | null;
|
|
84
86
|
error: string | null;
|
|
87
|
+
/** Phase 1 Candidate Supply Report ({fetched, selected, rejected, reasons}). */
|
|
88
|
+
candidate_report: Record<string, unknown> | null;
|
|
85
89
|
}
|
|
86
90
|
/**
|
|
87
91
|
* Business categories an operator actually asks about, counted per cell. The
|
|
@@ -254,6 +258,8 @@ export declare class SlotCoordinator {
|
|
|
254
258
|
* sole path to the submitted cell state.
|
|
255
259
|
*/
|
|
256
260
|
applyOutcome(slotId: string, targetId: string, outcome: TargetOutcome): void;
|
|
261
|
+
/** Persist the Phase 1 candidate-report funnel for a target cell. */
|
|
262
|
+
private persistCandidateReport;
|
|
257
263
|
/**
|
|
258
264
|
* Persist the normalized terminal reason (§terminal-reason) for a cell that
|
|
259
265
|
* just reached a terminal state. Never unwinds the run and never leaks raw
|
|
@@ -255,6 +255,7 @@ class SlotCoordinator {
|
|
|
255
255
|
switch (outcome.kind) {
|
|
256
256
|
case 'submitted':
|
|
257
257
|
this.database.slots.lockCellWork(slotId, targetId, outcome.workId, outcome.workType);
|
|
258
|
+
this.persistCandidateReport(slotId, targetId, outcome);
|
|
258
259
|
this.safeTransition(slotId, targetId, 'submitted');
|
|
259
260
|
return;
|
|
260
261
|
case 'stored':
|
|
@@ -262,22 +263,26 @@ class SlotCoordinator {
|
|
|
262
263
|
// cell, but labelled via the ledger-free 'submitted' aggregate state so
|
|
263
264
|
// download-only schedules do not rerun forever.
|
|
264
265
|
this.database.slots.lockCellWork(slotId, targetId, outcome.workId, outcome.workType);
|
|
266
|
+
this.persistCandidateReport(slotId, targetId, outcome);
|
|
265
267
|
this.safeTransition(slotId, targetId, 'submitted');
|
|
266
268
|
return;
|
|
267
269
|
case 'delivery_pending':
|
|
268
270
|
this.database.slots.lockCellWork(slotId, targetId, outcome.workId, outcome.workType);
|
|
271
|
+
this.persistCandidateReport(slotId, targetId, outcome);
|
|
269
272
|
this.safeTransition(slotId, targetId, 'delivery_pending');
|
|
270
273
|
return;
|
|
271
274
|
case 'no_candidate':
|
|
272
275
|
// Only terminal if the cell never locked a work; a locked work whose
|
|
273
276
|
// delivery is still pending must not be collapsed to no_candidate.
|
|
274
277
|
if (!cell.workId) {
|
|
278
|
+
this.persistCandidateReport(slotId, targetId, outcome);
|
|
275
279
|
this.safeTransition(slotId, targetId, 'no_candidate', outcome.reason);
|
|
276
280
|
this.persistTerminalReason(slotId, targetId, outcome);
|
|
277
281
|
}
|
|
278
282
|
return;
|
|
279
283
|
case 'duplicate':
|
|
280
284
|
this.database.slots.lockCellWork(slotId, targetId, outcome.workId, cell.workType ?? 'unknown');
|
|
285
|
+
this.persistCandidateReport(slotId, targetId, outcome);
|
|
281
286
|
this.safeTransition(slotId, targetId, 'duplicate', outcome.reason);
|
|
282
287
|
this.persistTerminalReason(slotId, targetId, outcome);
|
|
283
288
|
return;
|
|
@@ -286,13 +291,31 @@ class SlotCoordinator {
|
|
|
286
291
|
// Leave non-terminal (selected/delivery_pending) so a later trigger
|
|
287
292
|
// resumes the SAME work. Record the error without a terminal state.
|
|
288
293
|
this.database.slots.setCellError?.(slotId, targetId, outcome.error);
|
|
294
|
+
this.persistCandidateReport(slotId, targetId, outcome);
|
|
289
295
|
return;
|
|
290
296
|
}
|
|
297
|
+
this.persistCandidateReport(slotId, targetId, outcome);
|
|
291
298
|
this.safeTransition(slotId, targetId, 'failed', outcome.error);
|
|
292
299
|
this.persistTerminalReason(slotId, targetId, outcome);
|
|
293
300
|
return;
|
|
294
301
|
}
|
|
295
302
|
}
|
|
303
|
+
/** Persist the Phase 1 candidate-report funnel for a target cell. */
|
|
304
|
+
persistCandidateReport(slotId, targetId, outcome) {
|
|
305
|
+
const report = outcome.scan?.supply;
|
|
306
|
+
if (!report)
|
|
307
|
+
return;
|
|
308
|
+
try {
|
|
309
|
+
this.database.slots.setCellCandidateReport(slotId, targetId, report);
|
|
310
|
+
}
|
|
311
|
+
catch (error) {
|
|
312
|
+
logger_1.logger.debug('Failed to persist candidate report', {
|
|
313
|
+
slot: slotId,
|
|
314
|
+
target: targetId,
|
|
315
|
+
error: error instanceof Error ? error.message : String(error),
|
|
316
|
+
});
|
|
317
|
+
}
|
|
318
|
+
}
|
|
296
319
|
/**
|
|
297
320
|
* Persist the normalized terminal reason (§terminal-reason) for a cell that
|
|
298
321
|
* just reached a terminal state. Never unwinds the run and never leaks raw
|
|
@@ -444,6 +467,7 @@ class SlotCoordinator {
|
|
|
444
467
|
error: c.lastError,
|
|
445
468
|
terminalReasonCode: c.terminalReasonCode,
|
|
446
469
|
terminalReasonMessage: c.terminalReasonMessage,
|
|
470
|
+
candidateReport: c.candidateReport,
|
|
447
471
|
}));
|
|
448
472
|
// Rolled up AFTER the slot row carries its terminal status/completed_at, so
|
|
449
473
|
// the outcome reports the durable timestamps rather than a fresh clock read.
|
|
@@ -493,6 +517,7 @@ class SlotCoordinator {
|
|
|
493
517
|
error: cell.lastError,
|
|
494
518
|
terminal_reason_code: cell.terminalReasonCode,
|
|
495
519
|
terminal_reason_message: cell.terminalReasonMessage,
|
|
520
|
+
candidate_report: cell.candidateReport,
|
|
496
521
|
};
|
|
497
522
|
});
|
|
498
523
|
// A fully-submitted slot has no non-submitted cell at all, and reporting
|
|
@@ -611,6 +636,7 @@ class SlotCoordinator {
|
|
|
611
636
|
status: c.status,
|
|
612
637
|
workId: c.workId,
|
|
613
638
|
error: c.lastError,
|
|
639
|
+
candidateReport: c.candidateReport,
|
|
614
640
|
}));
|
|
615
641
|
const slotRec = this.database.slots.getSlot(slotId);
|
|
616
642
|
return {
|
|
@@ -96,6 +96,46 @@ export type CandidateAttempt =
|
|
|
96
96
|
kind: 'skipped';
|
|
97
97
|
skip: CandidateSkip;
|
|
98
98
|
};
|
|
99
|
+
/**
|
|
100
|
+
* Upstream candidate-supply funnel for one target run (topic pipeline).
|
|
101
|
+
*
|
|
102
|
+
* Deliberately an open `reasons` bucket, NOT a fixed list of columns:
|
|
103
|
+
* illustration and novel filter chains differ, and future TopicProfile /
|
|
104
|
+
* CandidateInventory phases must be able to extend the taxonomy without a
|
|
105
|
+
* breaking schema change.
|
|
106
|
+
*/
|
|
107
|
+
export interface CandidateSupplyReason {
|
|
108
|
+
/** Stable machine-readable reason code, e.g. `duplicate`, `ai_filtered`. */
|
|
109
|
+
code: string;
|
|
110
|
+
count: number;
|
|
111
|
+
}
|
|
112
|
+
/** Candidate-supply observability snapshot (Phase 1 Candidate Report). */
|
|
113
|
+
export interface CandidateSupplyReport {
|
|
114
|
+
/** Total works surfaced by the topic search before any filtering. */
|
|
115
|
+
fetched: number;
|
|
116
|
+
/** Candidates that survived upload selection (the pool handed to the scan). */
|
|
117
|
+
selected: number;
|
|
118
|
+
/** Count rejected by all filters (fetched - selected is a stable invariant). */
|
|
119
|
+
rejected: number;
|
|
120
|
+
/** Why candidates were rejected, by reason code (extensible). */
|
|
121
|
+
reasons: CandidateSupplyReason[];
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* A freshly-harvested upstream funnel with no candidates selected yet.
|
|
125
|
+
*/
|
|
126
|
+
export declare function emptyCandidateSupplyReport(): CandidateSupplyReport;
|
|
127
|
+
/**
|
|
128
|
+
* Fold two supply snapshots of the SAME logical target together, so a
|
|
129
|
+
* multi-day lookback scan reports the whole funnel rather than only its last
|
|
130
|
+
* day. Reasons are summed by code.
|
|
131
|
+
*/
|
|
132
|
+
export declare function mergeCandidateSupplyReports(first: CandidateSupplyReport | undefined, second: CandidateSupplyReport): CandidateSupplyReport;
|
|
133
|
+
/**
|
|
134
|
+
* Fold the scan-level (download-time) candidate skips into the upstream
|
|
135
|
+
* candidate report. `selected` is reduced by every usable candidate actually
|
|
136
|
+
* offered to the scan, and the skip reasons join the upstream reasons.
|
|
137
|
+
*/
|
|
138
|
+
export declare function withScanSkips(report: CandidateSupplyReport | undefined, scan: CandidateScanSummary | undefined): CandidateSupplyReport | undefined;
|
|
99
139
|
/**
|
|
100
140
|
* Bookkeeping for a target's bounded candidate scan. Attached to the terminal
|
|
101
141
|
* target outcome so the run can state exactly one of:
|
|
@@ -122,6 +162,12 @@ export interface CandidateScanSummary {
|
|
|
122
162
|
skipped: CandidateSkip[];
|
|
123
163
|
/** Job-level outages observed while scanning (never a candidate verdict). */
|
|
124
164
|
outages: JobLevelOutage[];
|
|
165
|
+
/**
|
|
166
|
+
* Upstream candidate-supply funnel that produced this scan, when the target
|
|
167
|
+
* ran through the topic pipeline (illustration or novel). Optional so ranking
|
|
168
|
+
* / search targets without a topic funnel keep working unchanged.
|
|
169
|
+
*/
|
|
170
|
+
supply?: CandidateSupplyReport;
|
|
125
171
|
}
|
|
126
172
|
/**
|
|
127
173
|
* A scan that attempted nothing. The real pipeline always reports its own scan;
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.TERMINAL_REASON_MESSAGES = void 0;
|
|
4
|
+
exports.emptyCandidateSupplyReport = emptyCandidateSupplyReport;
|
|
5
|
+
exports.mergeCandidateSupplyReports = mergeCandidateSupplyReports;
|
|
6
|
+
exports.withScanSkips = withScanSkips;
|
|
4
7
|
exports.emptyCandidateScan = emptyCandidateScan;
|
|
5
8
|
exports.isTerminalOutcome = isTerminalOutcome;
|
|
6
9
|
exports.isSelectedAttempt = isSelectedAttempt;
|
|
@@ -13,6 +16,60 @@ exports.classifyCandidateFailure = classifyCandidateFailure;
|
|
|
13
16
|
exports.classifyJobLevelOutage = classifyJobLevelOutage;
|
|
14
17
|
exports.operationalReasonForCode = operationalReasonForCode;
|
|
15
18
|
exports.terminalReasonFor = terminalReasonFor;
|
|
19
|
+
/**
|
|
20
|
+
* A freshly-harvested upstream funnel with no candidates selected yet.
|
|
21
|
+
*/
|
|
22
|
+
function emptyCandidateSupplyReport() {
|
|
23
|
+
return { fetched: 0, selected: 0, rejected: 0, reasons: [] };
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Fold two supply snapshots of the SAME logical target together, so a
|
|
27
|
+
* multi-day lookback scan reports the whole funnel rather than only its last
|
|
28
|
+
* day. Reasons are summed by code.
|
|
29
|
+
*/
|
|
30
|
+
function mergeCandidateSupplyReports(first, second) {
|
|
31
|
+
if (!first)
|
|
32
|
+
return second;
|
|
33
|
+
const byCode = new Map();
|
|
34
|
+
for (const r of [...first.reasons, ...second.reasons]) {
|
|
35
|
+
byCode.set(r.code, (byCode.get(r.code) ?? 0) + r.count);
|
|
36
|
+
}
|
|
37
|
+
const reasons = [...byCode.entries()]
|
|
38
|
+
.map(([code, count]) => ({ code, count }))
|
|
39
|
+
.filter((r) => r.count > 0)
|
|
40
|
+
.sort((a, b) => b.count - a.count);
|
|
41
|
+
return {
|
|
42
|
+
fetched: first.fetched + second.fetched,
|
|
43
|
+
selected: first.selected + second.selected,
|
|
44
|
+
rejected: Math.max(0, first.fetched + second.fetched - (first.selected + second.selected)),
|
|
45
|
+
reasons,
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Fold the scan-level (download-time) candidate skips into the upstream
|
|
50
|
+
* candidate report. `selected` is reduced by every usable candidate actually
|
|
51
|
+
* offered to the scan, and the skip reasons join the upstream reasons.
|
|
52
|
+
*/
|
|
53
|
+
function withScanSkips(report, scan) {
|
|
54
|
+
if (!scan || scan.skipped.length === 0)
|
|
55
|
+
return report;
|
|
56
|
+
const base = report ?? emptyCandidateSupplyReport();
|
|
57
|
+
const byCode = new Map(base.reasons.map((r) => [r.code, r.count]));
|
|
58
|
+
for (const skip of scan.skipped) {
|
|
59
|
+
byCode.set(skip.code, (byCode.get(skip.code) ?? 0) + 1);
|
|
60
|
+
}
|
|
61
|
+
const reasons = [...byCode.entries()]
|
|
62
|
+
.map(([code, count]) => ({ code, count }))
|
|
63
|
+
.filter((r) => r.count > 0)
|
|
64
|
+
.sort((a, b) => b.count - a.count);
|
|
65
|
+
const selected = Math.max(0, base.selected - scan.skipped.length);
|
|
66
|
+
return {
|
|
67
|
+
fetched: base.fetched,
|
|
68
|
+
selected,
|
|
69
|
+
rejected: Math.max(0, base.fetched - selected),
|
|
70
|
+
reasons,
|
|
71
|
+
};
|
|
72
|
+
}
|
|
16
73
|
/**
|
|
17
74
|
* A scan that attempted nothing. The real pipeline always reports its own scan;
|
|
18
75
|
* this is for callers/tests that have no candidate-level information, so they
|
|
@@ -130,6 +187,9 @@ function mergeScanSummaries(first, second) {
|
|
|
130
187
|
attempted: first.attempted + second.attempted,
|
|
131
188
|
skipped: [...first.skipped, ...second.skipped],
|
|
132
189
|
outages: [...new Set([...first.outages, ...second.outages])],
|
|
190
|
+
supply: second.supply
|
|
191
|
+
? mergeCandidateSupplyReports(first.supply, second.supply)
|
|
192
|
+
: first.supply,
|
|
133
193
|
};
|
|
134
194
|
}
|
|
135
195
|
const SQLITE_OUTAGE = /sqlite|database (?:is )?locked|unable to open database|no such table|disk i\/o error/i;
|
|
@@ -135,6 +135,7 @@ class DatabaseMigration {
|
|
|
135
135
|
-- code plus a user-facing business message, durable across restarts.
|
|
136
136
|
terminal_reason_code TEXT,
|
|
137
137
|
terminal_reason_message TEXT,
|
|
138
|
+
candidate_report TEXT,
|
|
138
139
|
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
139
140
|
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
140
141
|
completed_at DATETIME,
|
|
@@ -287,6 +288,9 @@ class DatabaseMigration {
|
|
|
287
288
|
if (!itemCols.includes('terminal_reason_message')) {
|
|
288
289
|
columnAlters.push(`ALTER TABLE schedule_slot_items ADD COLUMN terminal_reason_message TEXT`);
|
|
289
290
|
}
|
|
291
|
+
if (!itemCols.includes('candidate_report')) {
|
|
292
|
+
columnAlters.push(`ALTER TABLE schedule_slot_items ADD COLUMN candidate_report TEXT`);
|
|
293
|
+
}
|
|
290
294
|
// Create indexes for better query performance
|
|
291
295
|
const indexes = [
|
|
292
296
|
`CREATE INDEX IF NOT EXISTS idx_downloads_pixiv_id_type ON downloads(pixiv_id, type)`,
|
|
@@ -62,6 +62,11 @@ export interface SlotItemRecord {
|
|
|
62
62
|
terminalReasonCode: string | null;
|
|
63
63
|
/** User-facing business message for the terminal reason. */
|
|
64
64
|
terminalReasonMessage: string | null;
|
|
65
|
+
/**
|
|
66
|
+
* Candidate Supply Report (Phase 1), persisted as JSON on the same cell so
|
|
67
|
+
* the durable outcome and the review-group message can never disagree.
|
|
68
|
+
*/
|
|
69
|
+
candidateReport: Record<string, unknown> | null;
|
|
65
70
|
createdAt: string;
|
|
66
71
|
updatedAt: string;
|
|
67
72
|
completedAt: string | null;
|
|
@@ -172,6 +177,12 @@ export declare class SlotRepository extends BaseRepository {
|
|
|
172
177
|
* a later reasoned verdict.
|
|
173
178
|
*/
|
|
174
179
|
setCellTerminalReason(slotId: string, targetId: string, code: string, message: string): void;
|
|
180
|
+
/**
|
|
181
|
+
* Persist the Phase 1 candidate-supply funnel for a cell (JSON). Idempotent;
|
|
182
|
+
* recovery re-rolls overwrite with the newer observed funnel. A malformed
|
|
183
|
+
* payload must never block the terminal transition, so it is sanitised here.
|
|
184
|
+
*/
|
|
185
|
+
setCellCandidateReport(slotId: string, targetId: string, report: Record<string, unknown> | null): void;
|
|
175
186
|
/**
|
|
176
187
|
* Transition a cell with FSM validation. Never downgrades a confirmed cell;
|
|
177
188
|
* an illegal transition throws rather than silently corrupting state.
|
|
@@ -266,6 +266,22 @@ class SlotRepository extends BaseRepository_1.BaseRepository {
|
|
|
266
266
|
WHERE slot_id = @slotId AND target_id = @targetId`)
|
|
267
267
|
.run({ slotId, targetId, code: code.slice(0, 64), message: message.slice(0, 400) });
|
|
268
268
|
}
|
|
269
|
+
/**
|
|
270
|
+
* Persist the Phase 1 candidate-supply funnel for a cell (JSON). Idempotent;
|
|
271
|
+
* recovery re-rolls overwrite with the newer observed funnel. A malformed
|
|
272
|
+
* payload must never block the terminal transition, so it is sanitised here.
|
|
273
|
+
*/
|
|
274
|
+
setCellCandidateReport(slotId, targetId, report) {
|
|
275
|
+
const json = report == null ? null : JSON.stringify(report);
|
|
276
|
+
if (json != null && json.length > 4000) {
|
|
277
|
+
throw new Error('candidate_report exceeds 4000 chars');
|
|
278
|
+
}
|
|
279
|
+
this.db
|
|
280
|
+
.prepare(`UPDATE schedule_slot_items
|
|
281
|
+
SET candidate_report = @report, updated_at = CURRENT_TIMESTAMP
|
|
282
|
+
WHERE slot_id = @slotId AND target_id = @targetId`)
|
|
283
|
+
.run({ slotId, targetId, report: json });
|
|
284
|
+
}
|
|
269
285
|
/**
|
|
270
286
|
* Transition a cell with FSM validation. Never downgrades a confirmed cell;
|
|
271
287
|
* an illegal transition throws rather than silently corrupting state.
|
|
@@ -441,6 +457,7 @@ class SlotRepository extends BaseRepository_1.BaseRepository {
|
|
|
441
457
|
lastError: row.last_error,
|
|
442
458
|
terminalReasonCode: row.terminal_reason_code ?? null,
|
|
443
459
|
terminalReasonMessage: row.terminal_reason_message ?? null,
|
|
460
|
+
candidateReport: row.candidate_report ? JSON.parse(row.candidate_report) : null,
|
|
444
461
|
createdAt: row.created_at,
|
|
445
462
|
updatedAt: row.updated_at,
|
|
446
463
|
completedAt: row.completed_at,
|
|
@@ -9,6 +9,8 @@ export interface TopicSelection {
|
|
|
9
9
|
dedupedCount: number;
|
|
10
10
|
acceptedCount: number;
|
|
11
11
|
aiExcludedCount: number;
|
|
12
|
+
/** Works seen more than once across the topic tag space (recorded, dropped). */
|
|
13
|
+
duplicateRemovedCount: number;
|
|
12
14
|
}
|
|
13
15
|
/**
|
|
14
16
|
* Resolves a topic to a tag space, collects that day's works across the tags,
|
|
@@ -50,6 +50,7 @@ class TopicPipeline {
|
|
|
50
50
|
const byId = new Map();
|
|
51
51
|
let rawCount = 0;
|
|
52
52
|
let aiExcludedCount = 0;
|
|
53
|
+
let duplicateRemovedCount = 0;
|
|
53
54
|
const tagNames = space.tags.map((t) => t.name);
|
|
54
55
|
for (let i = 0; i < tagNames.length; i++) {
|
|
55
56
|
if (byId.size >= maxCandidates)
|
|
@@ -65,8 +66,10 @@ class TopicPipeline {
|
|
|
65
66
|
aiExcludedCount += 1;
|
|
66
67
|
continue;
|
|
67
68
|
}
|
|
68
|
-
if (byId.has(work.id))
|
|
69
|
+
if (byId.has(work.id)) {
|
|
70
|
+
duplicateRemovedCount += 1;
|
|
69
71
|
continue;
|
|
72
|
+
}
|
|
70
73
|
byId.set(work.id, { work, candidate: this.toCandidate(work, contentType) });
|
|
71
74
|
if (byId.size >= maxCandidates)
|
|
72
75
|
break;
|
|
@@ -107,6 +110,7 @@ class TopicPipeline {
|
|
|
107
110
|
dedupedCount,
|
|
108
111
|
acceptedCount: accepted.length,
|
|
109
112
|
aiExcludedCount,
|
|
113
|
+
duplicateRemovedCount,
|
|
110
114
|
},
|
|
111
115
|
};
|
|
112
116
|
}
|
package/dist/version.js
CHANGED
|
@@ -2,5 +2,5 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.BUILD = void 0;
|
|
4
4
|
// GENERATED by scripts/write-version.js — do not edit manually.
|
|
5
|
-
exports.BUILD = { version: '2.
|
|
5
|
+
exports.BUILD = { version: '2.29.0', commit: '13e234a69cd2' };
|
|
6
6
|
//# sourceMappingURL=version.js.map
|
package/dist/webui/package.json
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pixivflow",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.29.0",
|
|
4
4
|
"description": "🎨 Pixiv 下载、筛选与自动收集工具 - 批量下载插画和小说、按标签/热度/日期筛选、定时任务与可靠 HTTP 交付 | Pixiv downloader and automation toolkit with filtering, scheduling and reliable HTTP delivery",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|