pixivflow 2.28.1 → 2.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/scheduler-runtime.js +1 -0
- package/dist/config/types.d.ts +31 -0
- package/dist/config/validation.js +35 -0
- package/dist/delivery/DeliveryService.d.ts +1 -0
- package/dist/delivery/HttpMultipartDelivery.js +1 -0
- package/dist/delivery/types.d.ts +2 -0
- package/dist/download/handlers/IllustrationTargetHandler.d.ts +13 -0
- package/dist/download/handlers/IllustrationTargetHandler.js +99 -3
- package/dist/download/handlers/NovelTargetHandler.d.ts +11 -0
- package/dist/download/handlers/NovelTargetHandler.js +92 -3
- package/dist/download/inventory.d.ts +29 -0
- package/dist/download/inventory.js +80 -0
- package/dist/interfaces/IDatabase.d.ts +3 -0
- package/dist/notification/NotificationPolicy.d.ts +1 -0
- package/dist/notification/NotificationPolicy.js +1 -0
- package/dist/package.json +1 -1
- package/dist/scheduler/SlotCoordinator.d.ts +6 -0
- package/dist/scheduler/SlotCoordinator.js +26 -0
- package/dist/scheduler/TargetOutcome.d.ts +59 -0
- package/dist/scheduler/TargetOutcome.js +62 -0
- package/dist/storage/Database.d.ts +4 -0
- package/dist/storage/Database.js +7 -0
- package/dist/storage/DatabaseMigration.js +24 -0
- package/dist/storage/repositories/CandidateInventoryRepository.d.ts +51 -0
- package/dist/storage/repositories/CandidateInventoryRepository.js +129 -0
- package/dist/storage/repositories/SlotRepository.d.ts +11 -0
- package/dist/storage/repositories/SlotRepository.js +17 -0
- package/dist/topic/TopicPipeline.d.ts +2 -0
- package/dist/topic/TopicPipeline.js +5 -1
- package/dist/version.js +1 -1
- package/dist/webui/package.json +1 -1
- package/package.json +1 -1
package/dist/config/types.d.ts
CHANGED
|
@@ -27,6 +27,35 @@ export interface CandidateCollectionConfig {
|
|
|
27
27
|
/** Minimum metadata-topic score to survive filtering (default 0.35). */
|
|
28
28
|
minMetadataScore?: number;
|
|
29
29
|
}
|
|
30
|
+
/** Phase 4: topic-supply strategy (per-target, optional, default off). */
|
|
31
|
+
export interface TopicProfileStrategyConfig {
|
|
32
|
+
/** Relative weight for publishing fresher works first (0..1, default 0.4). */
|
|
33
|
+
freshnessWeight?: number;
|
|
34
|
+
/** Relative weight for publishing higher-popularity works first (0..1, default 0.3). */
|
|
35
|
+
popularityWeight?: number;
|
|
36
|
+
}
|
|
37
|
+
/** Phase 5: durable candidate inventory ("待发池") for a target. */
|
|
38
|
+
export interface CandidateInventoryConfig {
|
|
39
|
+
/** Master switch. Default false: no collection, no fallback, no new columns read. */
|
|
40
|
+
enabled?: boolean;
|
|
41
|
+
/** Max age (days) a pending candidate may stay usable for fallback. Default 30. */
|
|
42
|
+
maxAgeDays?: number;
|
|
43
|
+
/** Max pending rows retained per (topic,target). Default 20. */
|
|
44
|
+
reserveSize?: number;
|
|
45
|
+
/** When the fresh/lookback scan is empty, publish oldest pending inventory. Default true. */
|
|
46
|
+
fallback?: boolean;
|
|
47
|
+
}
|
|
48
|
+
/** Phase 4: TopicProfile (seed + related tags + supply strategy). */
|
|
49
|
+
export interface TopicProfileConfig {
|
|
50
|
+
/** Primary seed tag(s); first item is the canonical topic when topic is unset. */
|
|
51
|
+
primary?: string[];
|
|
52
|
+
/** Related tags that strengthen topic membership evidence. */
|
|
53
|
+
related?: string[];
|
|
54
|
+
/** Ranking strategy knobs (Phase 4; wired after CandidateInventory). */
|
|
55
|
+
strategy?: TopicProfileStrategyConfig;
|
|
56
|
+
/** CandidateInventory policy (Phase 5). */
|
|
57
|
+
inventory?: CandidateInventoryConfig;
|
|
58
|
+
}
|
|
30
59
|
/** Behaviour when a target cannot produce the requested number of works. */
|
|
31
60
|
export interface NoMatchPolicyConfig {
|
|
32
61
|
/**
|
|
@@ -178,6 +207,8 @@ export interface TargetConfig {
|
|
|
178
207
|
candidateCollection?: CandidateCollectionConfig;
|
|
179
208
|
/** Bounded fallback and notification policy for an empty result. */
|
|
180
209
|
noMatchPolicy?: NoMatchPolicyConfig;
|
|
210
|
+
/** Phase 4/5: topic supply profile + candidate inventory policy (all optional). */
|
|
211
|
+
topicProfile?: TopicProfileConfig;
|
|
181
212
|
/**
|
|
182
213
|
* Ranking mode (only used when mode='ranking')
|
|
183
214
|
* - 'day': Daily ranking
|
|
@@ -240,6 +240,41 @@ function validateConfig(config, location, databasePath) {
|
|
|
240
240
|
if (target.noMatchPolicy?.notify !== undefined && typeof target.noMatchPolicy.notify !== 'boolean') {
|
|
241
241
|
errors.push(`targets[${index}].noMatchPolicy.notify: Must be a boolean`);
|
|
242
242
|
}
|
|
243
|
+
if (target.topicProfile) {
|
|
244
|
+
const tp = target.topicProfile;
|
|
245
|
+
if (tp.primary !== undefined && (!Array.isArray(tp.primary) || tp.primary.length === 0 || tp.primary.some((x) => typeof x !== 'string' || !x.trim()))) {
|
|
246
|
+
errors.push(`targets[${index}].topicProfile.primary: Must be a non-empty string array`);
|
|
247
|
+
}
|
|
248
|
+
if (tp.related !== undefined && (!Array.isArray(tp.related) || tp.related.some((x) => typeof x !== 'string' || !x.trim()))) {
|
|
249
|
+
errors.push(`targets[${index}].topicProfile.related: Must be a string array`);
|
|
250
|
+
}
|
|
251
|
+
const st = tp.strategy;
|
|
252
|
+
if (st) {
|
|
253
|
+
for (const field of ['freshnessWeight', 'popularityWeight']) {
|
|
254
|
+
const value = st[field];
|
|
255
|
+
if (value !== undefined && (typeof value !== 'number' || value < 0 || value > 1)) {
|
|
256
|
+
errors.push(`targets[${index}].topicProfile.strategy.${field}: Must be a number between 0 and 1`);
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
const inv = tp.inventory;
|
|
261
|
+
if (inv) {
|
|
262
|
+
if (inv.enabled !== undefined && typeof inv.enabled !== 'boolean') {
|
|
263
|
+
errors.push(`targets[${index}].topicProfile.inventory.enabled: Must be a boolean`);
|
|
264
|
+
}
|
|
265
|
+
const bound = (field) => {
|
|
266
|
+
const value = inv[field];
|
|
267
|
+
if (value !== undefined && (typeof value !== 'number' || !Number.isInteger(value) || value < 1 || value > 365)) {
|
|
268
|
+
errors.push(`targets[${index}].topicProfile.inventory.${field}: Must be an integer between 1 and 365`);
|
|
269
|
+
}
|
|
270
|
+
};
|
|
271
|
+
bound('maxAgeDays');
|
|
272
|
+
bound('reserveSize');
|
|
273
|
+
if (inv.fallback !== undefined && typeof inv.fallback !== 'boolean') {
|
|
274
|
+
errors.push(`targets[${index}].topicProfile.inventory.fallback: Must be a boolean`);
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
}
|
|
243
278
|
if (target.noMatchPolicy?.notify === true) {
|
|
244
279
|
const deliveryTarget = target.delivery?.target?.trim();
|
|
245
280
|
const notifyTarget = deliveryTarget ? config.delivery?.targets?.[deliveryTarget] : undefined;
|
|
@@ -176,6 +176,7 @@ class HttpMultipartDelivery {
|
|
|
176
176
|
stage: t.stage ?? null,
|
|
177
177
|
retryable: t.retryable ?? null,
|
|
178
178
|
operator_hint: t.operator_hint ?? null,
|
|
179
|
+
candidate_report: t.candidate_report ?? null,
|
|
179
180
|
})),
|
|
180
181
|
}
|
|
181
182
|
: { text: request.text, idempotency_key: request.idempotencyKey };
|
package/dist/delivery/types.d.ts
CHANGED
|
@@ -146,6 +146,8 @@ export interface DeliveryNotificationRequest {
|
|
|
146
146
|
/** Whether a later/manual attempt may succeed; terminal does not mean auto-retry pending. */
|
|
147
147
|
retryable?: boolean | null;
|
|
148
148
|
operator_hint?: string | null;
|
|
149
|
+
/** Phase 1 Candidate Supply Report ({fetched, selected, rejected, reasons}). */
|
|
150
|
+
candidate_report?: Record<string, unknown> | null;
|
|
149
151
|
}>;
|
|
150
152
|
};
|
|
151
153
|
}
|
|
@@ -25,6 +25,11 @@ export declare class IllustrationTargetHandler {
|
|
|
25
25
|
* explicit verdict instead of an ambiguous "completed".
|
|
26
26
|
*/
|
|
27
27
|
private scan;
|
|
28
|
+
/**
|
|
29
|
+
* Upstream topic-supply funnel accumulated across a lookback scan (Phase 1
|
|
30
|
+
* Candidate Report: fetched/selected/rejected + reasons).
|
|
31
|
+
*/
|
|
32
|
+
private supplyRep;
|
|
28
33
|
/**
|
|
29
34
|
* Global candidate-scan bound (`download.candidateScanLimit`), supplied by
|
|
30
35
|
* DownloadManager. This is what lets the FETCH stage ask for more than one
|
|
@@ -63,6 +68,14 @@ export declare class IllustrationTargetHandler {
|
|
|
63
68
|
private fetchIllustrations;
|
|
64
69
|
private fetchTopicIllustrations;
|
|
65
70
|
private handleTopicWithLookback;
|
|
71
|
+
/**
|
|
72
|
+
* Phase 5 fallback: claim oldest pending candidate(s) from the durable
|
|
73
|
+
* 待发池 and try to publish them exactly like a normal day's pipeline run.
|
|
74
|
+
* Returns true when at least one work reached a delivery/stored outcome.
|
|
75
|
+
*/
|
|
76
|
+
private tryInventoryFallback;
|
|
77
|
+
/** Mark inventory rows whose work reached a delivery/stored outcome. */
|
|
78
|
+
private markInventorySubmittedFromOutcomes;
|
|
66
79
|
private resolveTopicDay;
|
|
67
80
|
private shiftDay;
|
|
68
81
|
private fetchRankingIllustrations;
|
|
@@ -12,6 +12,7 @@ const observability_1 = require("../../observability");
|
|
|
12
12
|
const context_1 = require("../../observability/context");
|
|
13
13
|
const DownloadPlanner_1 = require("../plan/DownloadPlanner");
|
|
14
14
|
const deliveryContext_1 = require("./deliveryContext");
|
|
15
|
+
const inventory_1 = require("../inventory");
|
|
15
16
|
class IllustrationTargetHandler {
|
|
16
17
|
client;
|
|
17
18
|
database;
|
|
@@ -29,6 +30,11 @@ class IllustrationTargetHandler {
|
|
|
29
30
|
* explicit verdict instead of an ambiguous "completed".
|
|
30
31
|
*/
|
|
31
32
|
scan = null;
|
|
33
|
+
/**
|
|
34
|
+
* Upstream topic-supply funnel accumulated across a lookback scan (Phase 1
|
|
35
|
+
* Candidate Report: fetched/selected/rejected + reasons).
|
|
36
|
+
*/
|
|
37
|
+
supplyRep = null;
|
|
32
38
|
/**
|
|
33
39
|
* Global candidate-scan bound (`download.candidateScanLimit`), supplied by
|
|
34
40
|
* DownloadManager. This is what lets the FETCH stage ask for more than one
|
|
@@ -68,6 +74,7 @@ class IllustrationTargetHandler {
|
|
|
68
74
|
async handle(target, execution) {
|
|
69
75
|
this.outcomes = [];
|
|
70
76
|
this.scan = null;
|
|
77
|
+
this.supplyRep = null;
|
|
71
78
|
this.execution = execution && (0, WorkIdentity_1.isSingleWorkCell)(target) ? execution : null;
|
|
72
79
|
// A cell that already owns a work is in RECOVERY, not in a new selection.
|
|
73
80
|
// Crash/shutdown recovery is not an intentional second run: running the
|
|
@@ -113,6 +120,9 @@ class IllustrationTargetHandler {
|
|
|
113
120
|
* scheduled slot report success after submitting nothing.
|
|
114
121
|
*/
|
|
115
122
|
summarize(target) {
|
|
123
|
+
if (this.scan && this.supplyRep) {
|
|
124
|
+
this.scan = { ...this.scan, supply: (0, TargetOutcome_1.withScanSkips)(this.supplyRep, this.scan) };
|
|
125
|
+
}
|
|
116
126
|
const scan = this.scan ?? undefined;
|
|
117
127
|
const submitted = this.outcomes.find((o) => o.kind === 'submitted');
|
|
118
128
|
if (submitted)
|
|
@@ -198,7 +208,8 @@ class IllustrationTargetHandler {
|
|
|
198
208
|
}
|
|
199
209
|
async fetchIllustrations(target, mode) {
|
|
200
210
|
if (mode === 'topic') {
|
|
201
|
-
|
|
211
|
+
const { works } = await this.fetchTopicIllustrations(target);
|
|
212
|
+
return works;
|
|
202
213
|
}
|
|
203
214
|
if (mode === 'ranking') {
|
|
204
215
|
return this.fetchRankingIllustrations(target);
|
|
@@ -223,7 +234,22 @@ class IllustrationTargetHandler {
|
|
|
223
234
|
const pipeline = this.topicPipelineFactory();
|
|
224
235
|
const { works, selection } = await pipeline.selectWorks(target, 'illustration', day, selectionLimit, target.topicDiscovery ?? {}, target.candidateCollection ?? {});
|
|
225
236
|
logger_1.logger.info(`Topic "${topic}" illustration: tags=${selection.resolvedTagCount} raw=${selection.rawCount} deduped=${selection.dedupedCount} aiExcluded=${selection.aiExcludedCount} accepted=${selection.acceptedCount} candidates=${works.length} target=${limit}`);
|
|
226
|
-
|
|
237
|
+
// Phase 5: harvest the accepted-but-not-yet-delivered works into the
|
|
238
|
+
// durable 待发池 (idempotent; only when the target opts in).
|
|
239
|
+
(0, inventory_1.recordInventoryCandidates)(this.database, target, topic, day, works, 'illustration');
|
|
240
|
+
// Candidate Supply Observability: fold this lookback day's upstream funnel
|
|
241
|
+
// into the target's accumulated Candidate Report.
|
|
242
|
+
const report = (0, TargetOutcome_1.emptyCandidateSupplyReport)();
|
|
243
|
+
report.fetched = selection.rawCount;
|
|
244
|
+
report.selected = selection.acceptedCount;
|
|
245
|
+
report.rejected = Math.max(0, selection.rawCount - selection.acceptedCount);
|
|
246
|
+
report.reasons = [
|
|
247
|
+
{ code: 'duplicate', count: selection.duplicateRemovedCount },
|
|
248
|
+
{ code: 'ai_filtered', count: selection.aiExcludedCount },
|
|
249
|
+
{ code: 'metadata_filtered', count: Math.max(0, selection.dedupedCount - selection.acceptedCount) },
|
|
250
|
+
].filter((r) => r.count > 0);
|
|
251
|
+
this.supplyRep = (0, TargetOutcome_1.mergeCandidateSupplyReports)(this.supplyRep ?? undefined, report);
|
|
252
|
+
return { works, selection };
|
|
227
253
|
}
|
|
228
254
|
async handleTopicWithLookback(target, displayTag) {
|
|
229
255
|
const requested = target.limit || 1;
|
|
@@ -241,13 +267,14 @@ class IllustrationTargetHandler {
|
|
|
241
267
|
fallbackOffset: offset,
|
|
242
268
|
});
|
|
243
269
|
}
|
|
244
|
-
const illusts = await this.fetchTopicIllustrations(attemptTarget);
|
|
270
|
+
const { works: illusts } = await this.fetchTopicIllustrations(attemptTarget);
|
|
245
271
|
const result = await this.pipeline.run(illusts, attemptTarget, 'illustration', (illust, tag) => this.downloadAndDeliver(illust, tag, attemptTarget));
|
|
246
272
|
// The lookback loop is one bounded scan: accumulate every day's
|
|
247
273
|
// skips/outages so the final verdict covers all candidates attempted.
|
|
248
274
|
this.scan = (0, TargetOutcome_1.mergeScanSummaries)(this.scan, result.scan);
|
|
249
275
|
if (result.downloaded > 0) {
|
|
250
276
|
this.handleDownloadResult(result, target, 'topic', illusts.length);
|
|
277
|
+
this.markInventorySubmittedFromOutcomes(target, (0, inventory_1.inventoryTopic)(target), 'illustration');
|
|
251
278
|
return;
|
|
252
279
|
}
|
|
253
280
|
if (this.scan && this.scan.outages.length > 0) {
|
|
@@ -256,6 +283,16 @@ class IllustrationTargetHandler {
|
|
|
256
283
|
return;
|
|
257
284
|
}
|
|
258
285
|
}
|
|
286
|
+
// Phase 5: when the fresh + lookback scan produced nothing, and the target
|
|
287
|
+
// opted into CandidateInventory, publish oldest pending reserve candidates.
|
|
288
|
+
const topic = (0, inventory_1.inventoryTopic)(target);
|
|
289
|
+
if (await this.tryInventoryFallback(target, topic, 'illustration')) {
|
|
290
|
+
return;
|
|
291
|
+
}
|
|
292
|
+
// Report how much reserve remains so the empty result is not a dead end.
|
|
293
|
+
if (this.supplyRep) {
|
|
294
|
+
this.supplyRep = (0, inventory_1.attachInventoryReport)(this.database, target, topic, this.supplyRep);
|
|
295
|
+
}
|
|
259
296
|
const scan = this.scan;
|
|
260
297
|
// An explicit no-eligible-candidate verdict needs something to have been
|
|
261
298
|
// considered. When the scan is empty (nothing surfaced at all) the richer
|
|
@@ -275,6 +312,65 @@ class IllustrationTargetHandler {
|
|
|
275
312
|
...(scan ? { scan } : {}),
|
|
276
313
|
});
|
|
277
314
|
}
|
|
315
|
+
/**
|
|
316
|
+
* Phase 5 fallback: claim oldest pending candidate(s) from the durable
|
|
317
|
+
* 待发池 and try to publish them exactly like a normal day's pipeline run.
|
|
318
|
+
* Returns true when at least one work reached a delivery/stored outcome.
|
|
319
|
+
*/
|
|
320
|
+
async tryInventoryFallback(target, topic, workType) {
|
|
321
|
+
const policy = (0, inventory_1.inventoryPolicy)(target);
|
|
322
|
+
if (!policy.enabled || policy.fallback === false)
|
|
323
|
+
return false;
|
|
324
|
+
const targetId = (0, inventory_1.inventoryTargetId)(target);
|
|
325
|
+
const repo = this.database.candidateInventory;
|
|
326
|
+
while (true) {
|
|
327
|
+
const row = repo.claimNext({ topic, targetId, reserveSize: policy.reserveSize, date: (0, pixiv_date_utils_1.getTodayDate)() });
|
|
328
|
+
if (!row)
|
|
329
|
+
break;
|
|
330
|
+
const id = Number(row.pixivId);
|
|
331
|
+
if (!Number.isFinite(id)) {
|
|
332
|
+
repo.markFiltered(row.pixivId, row.workType, topic, targetId);
|
|
333
|
+
continue;
|
|
334
|
+
}
|
|
335
|
+
let detail;
|
|
336
|
+
try {
|
|
337
|
+
detail = await this.client.getIllustration(id);
|
|
338
|
+
}
|
|
339
|
+
catch (error) {
|
|
340
|
+
this.logError(error, `Inventory candidate illustration ${id} could not be fetched`);
|
|
341
|
+
repo.markSelectedBackToPending(row.pixivId, row.workType, topic, targetId);
|
|
342
|
+
continue;
|
|
343
|
+
}
|
|
344
|
+
const attemptTarget = { ...target };
|
|
345
|
+
const result = await this.pipeline.run([detail], attemptTarget, 'illustration', (illust, tag) => this.downloadAndDeliver(illust, tag, attemptTarget));
|
|
346
|
+
this.scan = (0, TargetOutcome_1.mergeScanSummaries)(this.scan, result.scan);
|
|
347
|
+
if (result.downloaded > 0) {
|
|
348
|
+
this.handleDownloadResult(result, attemptTarget, 'topic', 1);
|
|
349
|
+
(0, inventory_1.markInventoryAttempt)(this.database, target, topic, row.pixivId, 'illustration', 'submitted');
|
|
350
|
+
return true;
|
|
351
|
+
}
|
|
352
|
+
if (this.scan && this.scan.outages.length > 0) {
|
|
353
|
+
// A dead token / dead network is not "no candidate": retry the same
|
|
354
|
+
// row on the next run instead of discarding it.
|
|
355
|
+
(0, inventory_1.markInventoryAttempt)(this.database, target, topic, row.pixivId, 'illustration', 'pending');
|
|
356
|
+
return false;
|
|
357
|
+
}
|
|
358
|
+
// The claimed work is no longer usable (deleted/private/duplicate): drop
|
|
359
|
+
// it permanently and try the next pending row.
|
|
360
|
+
(0, inventory_1.markInventoryAttempt)(this.database, target, topic, row.pixivId, 'illustration', 'filtered');
|
|
361
|
+
}
|
|
362
|
+
return false;
|
|
363
|
+
}
|
|
364
|
+
/** Mark inventory rows whose work reached a delivery/stored outcome. */
|
|
365
|
+
markInventorySubmittedFromOutcomes(target, topic, workType) {
|
|
366
|
+
if (!(0, inventory_1.inventoryPolicy)(target).enabled)
|
|
367
|
+
return;
|
|
368
|
+
for (const outcome of this.outcomes) {
|
|
369
|
+
if (outcome.kind === 'delivery_pending' || outcome.kind === 'submitted' || outcome.kind === 'stored') {
|
|
370
|
+
(0, inventory_1.markInventoryAttempt)(this.database, target, topic, outcome.workId, workType, 'submitted');
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
}
|
|
278
374
|
resolveTopicDay(target) {
|
|
279
375
|
return target.date === 'TODAY'
|
|
280
376
|
? (0, pixiv_date_utils_1.getTodayDate)()
|
|
@@ -24,6 +24,10 @@ export declare class NovelTargetHandler {
|
|
|
24
24
|
* explicit verdict instead of an ambiguous "completed".
|
|
25
25
|
*/
|
|
26
26
|
private scan;
|
|
27
|
+
/**
|
|
28
|
+
* Upstream topic-supply funnel accumulated across a lookback scan.
|
|
29
|
+
*/
|
|
30
|
+
private supplyRep;
|
|
27
31
|
/**
|
|
28
32
|
* Global candidate-scan bound (`download.candidateScanLimit`), supplied by
|
|
29
33
|
* DownloadManager. This is what lets the FETCH stage ask for more than one
|
|
@@ -63,6 +67,13 @@ export declare class NovelTargetHandler {
|
|
|
63
67
|
private fetchNovels;
|
|
64
68
|
private fetchTopicNovels;
|
|
65
69
|
private handleTopicWithLookback;
|
|
70
|
+
/**
|
|
71
|
+
* Phase 5 fallback: claim pending novel candidates and publish them into the
|
|
72
|
+
* existing aggregate day-scan (same idempotent pipeline as a normal day).
|
|
73
|
+
*/
|
|
74
|
+
private tryInventoryFallback;
|
|
75
|
+
/** Mark inventory rows whose work reached a delivery/stored outcome. */
|
|
76
|
+
private markInventorySubmittedFromOutcomes;
|
|
66
77
|
private resolveTopicDay;
|
|
67
78
|
private shiftDay;
|
|
68
79
|
private fetchRankingNovels;
|
|
@@ -12,6 +12,7 @@ const observability_1 = require("../../observability");
|
|
|
12
12
|
const context_1 = require("../../observability/context");
|
|
13
13
|
const DownloadPlanner_1 = require("../plan/DownloadPlanner");
|
|
14
14
|
const deliveryContext_1 = require("./deliveryContext");
|
|
15
|
+
const inventory_1 = require("../inventory");
|
|
15
16
|
const TelePressRichNovel_1 = require("../../delivery/TelePressRichNovel");
|
|
16
17
|
class NovelTargetHandler {
|
|
17
18
|
client;
|
|
@@ -29,6 +30,10 @@ class NovelTargetHandler {
|
|
|
29
30
|
* explicit verdict instead of an ambiguous "completed".
|
|
30
31
|
*/
|
|
31
32
|
scan = null;
|
|
33
|
+
/**
|
|
34
|
+
* Upstream topic-supply funnel accumulated across a lookback scan.
|
|
35
|
+
*/
|
|
36
|
+
supplyRep = null;
|
|
32
37
|
/**
|
|
33
38
|
* Global candidate-scan bound (`download.candidateScanLimit`), supplied by
|
|
34
39
|
* DownloadManager. This is what lets the FETCH stage ask for more than one
|
|
@@ -68,6 +73,7 @@ class NovelTargetHandler {
|
|
|
68
73
|
async handle(target, execution) {
|
|
69
74
|
this.outcomes = [];
|
|
70
75
|
this.scan = null;
|
|
76
|
+
this.supplyRep = null;
|
|
71
77
|
this.execution = execution && (0, WorkIdentity_1.isSingleWorkCell)(target) ? execution : null;
|
|
72
78
|
// A cell that already owns a work is in RECOVERY, not in a new selection.
|
|
73
79
|
// Crash/shutdown recovery is not an intentional second run: running the
|
|
@@ -126,6 +132,9 @@ class NovelTargetHandler {
|
|
|
126
132
|
* single-work RECOVERY path, whose cell identity is fixed.
|
|
127
133
|
*/
|
|
128
134
|
summarize() {
|
|
135
|
+
if (this.scan && this.supplyRep) {
|
|
136
|
+
this.scan = { ...this.scan, supply: (0, TargetOutcome_1.withScanSkips)(this.supplyRep, this.scan) };
|
|
137
|
+
}
|
|
129
138
|
const scan = this.scan ?? undefined;
|
|
130
139
|
const submitted = this.outcomes.find((o) => o.kind === 'submitted');
|
|
131
140
|
if (submitted)
|
|
@@ -200,7 +209,8 @@ class NovelTargetHandler {
|
|
|
200
209
|
}
|
|
201
210
|
async fetchNovels(target, mode) {
|
|
202
211
|
if (mode === 'topic') {
|
|
203
|
-
|
|
212
|
+
const { works } = await this.fetchTopicNovels(target);
|
|
213
|
+
return works;
|
|
204
214
|
}
|
|
205
215
|
if (mode === 'ranking') {
|
|
206
216
|
return this.fetchRankingNovels(target);
|
|
@@ -223,7 +233,17 @@ class NovelTargetHandler {
|
|
|
223
233
|
const pipeline = this.topicPipelineFactory();
|
|
224
234
|
const { works, selection } = await pipeline.selectWorks(target, 'novel', day, selectionLimit, target.topicDiscovery ?? {}, target.candidateCollection ?? {});
|
|
225
235
|
logger_1.logger.info(`Topic "${topic}" novel: tags=${selection.resolvedTagCount} raw=${selection.rawCount} deduped=${selection.dedupedCount} accepted=${selection.acceptedCount} candidates=${works.length} target=${limit}`);
|
|
226
|
-
|
|
236
|
+
(0, inventory_1.recordInventoryCandidates)(this.database, target, topic, day, works, 'novel');
|
|
237
|
+
const report = (0, TargetOutcome_1.emptyCandidateSupplyReport)();
|
|
238
|
+
report.fetched = selection.rawCount;
|
|
239
|
+
report.selected = selection.acceptedCount;
|
|
240
|
+
report.rejected = Math.max(0, selection.rawCount - selection.acceptedCount);
|
|
241
|
+
report.reasons = [
|
|
242
|
+
{ code: 'duplicate', count: selection.duplicateRemovedCount },
|
|
243
|
+
{ code: 'metadata_filtered', count: Math.max(0, selection.dedupedCount - selection.acceptedCount) },
|
|
244
|
+
].filter((r) => r.count > 0);
|
|
245
|
+
this.supplyRep = (0, TargetOutcome_1.mergeCandidateSupplyReports)(this.supplyRep ?? undefined, report);
|
|
246
|
+
return { works, selection };
|
|
227
247
|
}
|
|
228
248
|
async handleTopicWithLookback(target, displayTag) {
|
|
229
249
|
const requested = target.limit || 1;
|
|
@@ -253,7 +273,7 @@ class NovelTargetHandler {
|
|
|
253
273
|
fallbackOffset: offset,
|
|
254
274
|
});
|
|
255
275
|
}
|
|
256
|
-
const novels = await this.fetchTopicNovels(attemptTarget);
|
|
276
|
+
const { works: novels } = await this.fetchTopicNovels(attemptTarget);
|
|
257
277
|
totalFound += novels.length;
|
|
258
278
|
const result = await this.pipeline.run(novels, attemptTarget, 'novel', (novel, tag) => this.downloadAndDeliver(novel, tag, attemptTarget));
|
|
259
279
|
// The lookback loop is ONE bounded scan: accumulate every day's
|
|
@@ -263,15 +283,84 @@ class NovelTargetHandler {
|
|
|
263
283
|
aggregate.skipped += result.skipped;
|
|
264
284
|
aggregate.alreadyDownloaded += result.alreadyDownloaded;
|
|
265
285
|
aggregate.filteredOut += result.filteredOut;
|
|
286
|
+
if (result.downloaded > 0) {
|
|
287
|
+
this.markInventorySubmittedFromOutcomes(target, (0, inventory_1.inventoryTopic)(target), 'novel');
|
|
288
|
+
}
|
|
266
289
|
if (aggregate.scan.outages.length > 0) {
|
|
267
290
|
// A dead token / dead database / dead network is not "no matching
|
|
268
291
|
// novel": stop looking back and let the job fail/retry.
|
|
269
292
|
break;
|
|
270
293
|
}
|
|
271
294
|
}
|
|
295
|
+
const topic = (0, inventory_1.inventoryTopic)(target);
|
|
296
|
+
await this.tryInventoryFallback(target, topic, aggregate);
|
|
297
|
+
if (this.supplyRep) {
|
|
298
|
+
this.supplyRep = (0, inventory_1.attachInventoryReport)(this.database, target, topic, this.supplyRep);
|
|
299
|
+
}
|
|
272
300
|
this.scan = aggregate.scan;
|
|
273
301
|
await this.handleDownloadResult(aggregate, target, 'topic', totalFound, checkedDays);
|
|
274
302
|
}
|
|
303
|
+
/**
|
|
304
|
+
* Phase 5 fallback: claim pending novel candidates and publish them into the
|
|
305
|
+
* existing aggregate day-scan (same idempotent pipeline as a normal day).
|
|
306
|
+
*/
|
|
307
|
+
async tryInventoryFallback(target, topic, aggregate) {
|
|
308
|
+
const policy = (0, inventory_1.inventoryPolicy)(target);
|
|
309
|
+
if (!policy.enabled || policy.fallback === false)
|
|
310
|
+
return;
|
|
311
|
+
// Only a fully-empty fresh/lookback window may draw from the reserve; a
|
|
312
|
+
// partial day stays on fresh supply rather than padding beyond the limit.
|
|
313
|
+
if (aggregate.downloaded > 0)
|
|
314
|
+
return;
|
|
315
|
+
const targetId = (0, inventory_1.inventoryTargetId)(target);
|
|
316
|
+
const repo = this.database.candidateInventory;
|
|
317
|
+
while (aggregate.downloaded < (target.limit || 1)) {
|
|
318
|
+
const row = repo.claimNext({ topic, targetId, reserveSize: policy.reserveSize, date: (0, pixiv_date_utils_1.getTodayDate)() });
|
|
319
|
+
if (!row)
|
|
320
|
+
break;
|
|
321
|
+
const id = Number(row.pixivId);
|
|
322
|
+
if (!Number.isFinite(id)) {
|
|
323
|
+
repo.markFiltered(row.pixivId, row.workType, topic, targetId);
|
|
324
|
+
continue;
|
|
325
|
+
}
|
|
326
|
+
let detail;
|
|
327
|
+
try {
|
|
328
|
+
detail = await this.client.getNovelDetail(id);
|
|
329
|
+
}
|
|
330
|
+
catch (error) {
|
|
331
|
+
this.logError(error, `Inventory candidate novel ${id} could not be fetched`);
|
|
332
|
+
repo.markSelectedBackToPending(row.pixivId, row.workType, topic, targetId);
|
|
333
|
+
continue;
|
|
334
|
+
}
|
|
335
|
+
const attemptTarget = {
|
|
336
|
+
...target,
|
|
337
|
+
limit: (target.limit || 1) - aggregate.downloaded,
|
|
338
|
+
};
|
|
339
|
+
const result = await this.pipeline.run([detail], attemptTarget, 'novel', (novel, tag) => this.downloadAndDeliver(novel, tag, attemptTarget));
|
|
340
|
+
aggregate.scan = (0, TargetOutcome_1.mergeScanSummaries)(aggregate.scan, result.scan);
|
|
341
|
+
aggregate.downloaded += result.downloaded;
|
|
342
|
+
aggregate.skipped += result.skipped;
|
|
343
|
+
aggregate.alreadyDownloaded += result.alreadyDownloaded;
|
|
344
|
+
aggregate.filteredOut += result.filteredOut;
|
|
345
|
+
if (result.downloaded > 0) {
|
|
346
|
+
(0, inventory_1.markInventoryAttempt)(this.database, target, topic, row.pixivId, 'novel', 'submitted');
|
|
347
|
+
continue;
|
|
348
|
+
}
|
|
349
|
+
if (aggregate.scan.outages.length > 0)
|
|
350
|
+
return;
|
|
351
|
+
(0, inventory_1.markInventoryAttempt)(this.database, target, topic, row.pixivId, 'novel', 'filtered');
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
/** Mark inventory rows whose work reached a delivery/stored outcome. */
|
|
355
|
+
markInventorySubmittedFromOutcomes(target, topic, workType) {
|
|
356
|
+
if (!(0, inventory_1.inventoryPolicy)(target).enabled)
|
|
357
|
+
return;
|
|
358
|
+
for (const outcome of this.outcomes) {
|
|
359
|
+
if (outcome.kind === 'delivery_pending' || outcome.kind === 'submitted' || outcome.kind === 'stored') {
|
|
360
|
+
(0, inventory_1.markInventoryAttempt)(this.database, target, topic, outcome.workId, workType, 'submitted');
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
}
|
|
275
364
|
resolveTopicDay(target) {
|
|
276
365
|
return target.date === 'TODAY'
|
|
277
366
|
? (0, pixiv_date_utils_1.getTodayDate)()
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { TargetConfig } from '../config';
|
|
2
|
+
import type { IDatabase } from '../interfaces/IDatabase';
|
|
3
|
+
import type { CandidateSupplyReport } from '../scheduler/TargetOutcome';
|
|
4
|
+
/** Defaults and bounds for the optional Phase 5 inventory. */
|
|
5
|
+
export interface InventoryPolicy {
|
|
6
|
+
enabled: boolean;
|
|
7
|
+
maxAgeDays: number;
|
|
8
|
+
reserveSize: number;
|
|
9
|
+
fallback: boolean;
|
|
10
|
+
}
|
|
11
|
+
export declare function inventoryPolicy(target: TargetConfig): InventoryPolicy;
|
|
12
|
+
export declare function inventoryTopic(target: TargetConfig): string;
|
|
13
|
+
export declare function inventoryTargetId(target: TargetConfig): string;
|
|
14
|
+
/**
|
|
15
|
+
* Phase 5: after a topic scan yields eligible works that were NOT delivered by
|
|
16
|
+
* this run's lookback loop, upsert them as idle pending inventory. Works that
|
|
17
|
+
* are already recorded as downloaded/delivered are skipped by the caller
|
|
18
|
+
* upstream (selectWorks already removed download-history rows); this method is
|
|
19
|
+
* deliberately idempotent.
|
|
20
|
+
*/
|
|
21
|
+
export declare function recordInventoryCandidates(database: IDatabase, target: TargetConfig, topic: string, date: string, works: Array<{
|
|
22
|
+
id: number;
|
|
23
|
+
type?: string;
|
|
24
|
+
}>, workType: 'illustration' | 'novel'): void;
|
|
25
|
+
/** Enrich a candidate report with the durable pending-count snapshot. */
|
|
26
|
+
export declare function attachInventoryReport(database: IDatabase, target: TargetConfig, topic: string, report: CandidateSupplyReport): CandidateSupplyReport;
|
|
27
|
+
/** Marks a claimed inventory row after a delivery attempt. */
|
|
28
|
+
export declare function markInventoryAttempt(database: IDatabase, target: TargetConfig, topic: string, workId: string, workType: 'illustration' | 'novel', outcome: 'submitted' | 'filtered' | 'pending'): void;
|
|
29
|
+
//# sourceMappingURL=inventory.d.ts.map
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.inventoryPolicy = inventoryPolicy;
|
|
4
|
+
exports.inventoryTopic = inventoryTopic;
|
|
5
|
+
exports.inventoryTargetId = inventoryTargetId;
|
|
6
|
+
exports.recordInventoryCandidates = recordInventoryCandidates;
|
|
7
|
+
exports.attachInventoryReport = attachInventoryReport;
|
|
8
|
+
exports.markInventoryAttempt = markInventoryAttempt;
|
|
9
|
+
function inventoryPolicy(target) {
|
|
10
|
+
const cfg = target.topicProfile?.inventory;
|
|
11
|
+
return {
|
|
12
|
+
enabled: cfg?.enabled === true,
|
|
13
|
+
maxAgeDays: cfg?.maxAgeDays ?? 30,
|
|
14
|
+
reserveSize: Math.min(Math.max(cfg?.reserveSize ?? 20, 1), 100),
|
|
15
|
+
fallback: cfg?.fallback ?? true,
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
function inventoryTopic(target) {
|
|
19
|
+
return (target.topicProfile?.primary?.[0] ?? target.topic ?? '').trim();
|
|
20
|
+
}
|
|
21
|
+
function inventoryTargetId(target) {
|
|
22
|
+
return target.id ?? inventoryTopic(target);
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Phase 5: after a topic scan yields eligible works that were NOT delivered by
|
|
26
|
+
* this run's lookback loop, upsert them as idle pending inventory. Works that
|
|
27
|
+
* are already recorded as downloaded/delivered are skipped by the caller
|
|
28
|
+
* upstream (selectWorks already removed download-history rows); this method is
|
|
29
|
+
* deliberately idempotent.
|
|
30
|
+
*/
|
|
31
|
+
function recordInventoryCandidates(database, target, topic, date, works, workType) {
|
|
32
|
+
const policy = inventoryPolicy(target);
|
|
33
|
+
if (!policy.enabled)
|
|
34
|
+
return;
|
|
35
|
+
const targetId = inventoryTargetId(target);
|
|
36
|
+
for (const work of works) {
|
|
37
|
+
const id = String(work?.id);
|
|
38
|
+
if (!id)
|
|
39
|
+
continue;
|
|
40
|
+
database.candidateInventory.upsert({
|
|
41
|
+
pixivId: id,
|
|
42
|
+
workType,
|
|
43
|
+
topic,
|
|
44
|
+
targetId,
|
|
45
|
+
snapshot: work,
|
|
46
|
+
date,
|
|
47
|
+
maxAgeDays: policy.maxAgeDays,
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
/** Enrich a candidate report with the durable pending-count snapshot. */
|
|
52
|
+
function attachInventoryReport(database, target, topic, report) {
|
|
53
|
+
const policy = inventoryPolicy(target);
|
|
54
|
+
if (!policy.enabled)
|
|
55
|
+
return report;
|
|
56
|
+
const summary = database.candidateInventory.pendingSummary(topic, inventoryTargetId(target));
|
|
57
|
+
return {
|
|
58
|
+
...report,
|
|
59
|
+
inventory: {
|
|
60
|
+
pendingCount: summary.count,
|
|
61
|
+
reserveSize: policy.reserveSize,
|
|
62
|
+
maxAgeDays: policy.maxAgeDays,
|
|
63
|
+
oldestSeenDate: summary.oldestSeenDate,
|
|
64
|
+
},
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
/** Marks a claimed inventory row after a delivery attempt. */
|
|
68
|
+
function markInventoryAttempt(database, target, topic, workId, workType, outcome) {
|
|
69
|
+
if (!inventoryPolicy(target).enabled)
|
|
70
|
+
return;
|
|
71
|
+
const targetId = inventoryTargetId(target);
|
|
72
|
+
const repo = database.candidateInventory;
|
|
73
|
+
if (outcome === 'submitted')
|
|
74
|
+
repo.markSubmitted(workId, workType, topic, targetId);
|
|
75
|
+
else if (outcome === 'filtered')
|
|
76
|
+
repo.markFiltered(workId, workType, topic, targetId);
|
|
77
|
+
else
|
|
78
|
+
repo.markSelectedBackToPending(workId, workType, topic, targetId);
|
|
79
|
+
}
|
|
80
|
+
//# sourceMappingURL=inventory.js.map
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { AccessTokenStore, DownloadRecordInput, ExecutionStatus, SchedulerExecutionRecord } from '../storage/Database';
|
|
2
|
+
import { CandidateInventoryRepository } from '../storage/repositories/CandidateInventoryRepository';
|
|
2
3
|
/**
|
|
3
4
|
* Interface for database operations
|
|
4
5
|
* Provides abstraction for data persistence
|
|
@@ -123,5 +124,7 @@ export interface IDatabase {
|
|
|
123
124
|
};
|
|
124
125
|
countSince(botId: string | null, hours: number): number;
|
|
125
126
|
};
|
|
127
|
+
/** Phase 5 durable CandidateInventory (待发池). */
|
|
128
|
+
readonly candidateInventory: CandidateInventoryRepository;
|
|
126
129
|
}
|
|
127
130
|
//# sourceMappingURL=IDatabase.d.ts.map
|
|
@@ -46,6 +46,7 @@ export declare class NotificationPolicy {
|
|
|
46
46
|
error: string | null;
|
|
47
47
|
terminal_reason_code?: string | null;
|
|
48
48
|
reason?: string | null;
|
|
49
|
+
candidateReport?: Record<string, unknown> | null;
|
|
49
50
|
}>): void;
|
|
50
51
|
/**
|
|
51
52
|
* Delivery targets whose HTTP target declares the given outcome URL.
|