pixivflow 2.28.1 → 2.30.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/dist/commands/scheduler-runtime.js +1 -0
  2. package/dist/config/types.d.ts +31 -0
  3. package/dist/config/validation.js +35 -0
  4. package/dist/delivery/DeliveryService.d.ts +1 -0
  5. package/dist/delivery/HttpMultipartDelivery.js +1 -0
  6. package/dist/delivery/types.d.ts +2 -0
  7. package/dist/download/handlers/IllustrationTargetHandler.d.ts +13 -0
  8. package/dist/download/handlers/IllustrationTargetHandler.js +99 -3
  9. package/dist/download/handlers/NovelTargetHandler.d.ts +11 -0
  10. package/dist/download/handlers/NovelTargetHandler.js +92 -3
  11. package/dist/download/inventory.d.ts +29 -0
  12. package/dist/download/inventory.js +80 -0
  13. package/dist/interfaces/IDatabase.d.ts +3 -0
  14. package/dist/notification/NotificationPolicy.d.ts +1 -0
  15. package/dist/notification/NotificationPolicy.js +1 -0
  16. package/dist/package.json +1 -1
  17. package/dist/scheduler/SlotCoordinator.d.ts +6 -0
  18. package/dist/scheduler/SlotCoordinator.js +26 -0
  19. package/dist/scheduler/TargetOutcome.d.ts +59 -0
  20. package/dist/scheduler/TargetOutcome.js +62 -0
  21. package/dist/storage/Database.d.ts +4 -0
  22. package/dist/storage/Database.js +7 -0
  23. package/dist/storage/DatabaseMigration.js +24 -0
  24. package/dist/storage/repositories/CandidateInventoryRepository.d.ts +51 -0
  25. package/dist/storage/repositories/CandidateInventoryRepository.js +129 -0
  26. package/dist/storage/repositories/SlotRepository.d.ts +11 -0
  27. package/dist/storage/repositories/SlotRepository.js +17 -0
  28. package/dist/topic/TopicPipeline.d.ts +2 -0
  29. package/dist/topic/TopicPipeline.js +5 -1
  30. package/dist/version.js +1 -1
  31. package/dist/webui/package.json +1 -1
  32. package/package.json +1 -1
@@ -709,6 +709,7 @@ async function createSchedulerRuntime(configPathArg) {
709
709
  error: c.error ?? null,
710
710
  terminal_reason_code: c.terminalReasonCode ?? null,
711
711
  reason: c.terminalReasonMessage ?? null,
712
+ candidateReport: c.candidateReport ?? null,
712
713
  };
713
714
  }));
714
715
  }
@@ -27,6 +27,35 @@ export interface CandidateCollectionConfig {
27
27
  /** Minimum metadata-topic score to survive filtering (default 0.35). */
28
28
  minMetadataScore?: number;
29
29
  }
30
+ /** Phase 4: topic-supply strategy (per-target, optional, default off). */
31
+ export interface TopicProfileStrategyConfig {
32
+ /** Relative weight for publishing fresher works first (0..1, default 0.4). */
33
+ freshnessWeight?: number;
34
+ /** Relative weight for publishing higher-popularity works first (0..1, default 0.3). */
35
+ popularityWeight?: number;
36
+ }
37
+ /** Phase 5: durable candidate inventory ("待发池") for a target. */
38
+ export interface CandidateInventoryConfig {
39
+ /** Master switch. Default false: no collection, no fallback, no new columns read. */
40
+ enabled?: boolean;
41
+ /** Max age (days) a pending candidate may stay usable for fallback. Default 30. */
42
+ maxAgeDays?: number;
43
+ /** Max pending rows retained per (topic,target). Default 20. */
44
+ reserveSize?: number;
45
+ /** When the fresh/lookback scan is empty, publish oldest pending inventory. Default true. */
46
+ fallback?: boolean;
47
+ }
48
+ /** Phase 4: TopicProfile (seed + related tags + supply strategy). */
49
+ export interface TopicProfileConfig {
50
+ /** Primary seed tag(s); first item is the canonical topic when topic is unset. */
51
+ primary?: string[];
52
+ /** Related tags that strengthen topic membership evidence. */
53
+ related?: string[];
54
+ /** Ranking strategy knobs (Phase 4; wired after CandidateInventory). */
55
+ strategy?: TopicProfileStrategyConfig;
56
+ /** CandidateInventory policy (Phase 5). */
57
+ inventory?: CandidateInventoryConfig;
58
+ }
30
59
  /** Behaviour when a target cannot produce the requested number of works. */
31
60
  export interface NoMatchPolicyConfig {
32
61
  /**
@@ -178,6 +207,8 @@ export interface TargetConfig {
178
207
  candidateCollection?: CandidateCollectionConfig;
179
208
  /** Bounded fallback and notification policy for an empty result. */
180
209
  noMatchPolicy?: NoMatchPolicyConfig;
210
+ /** Phase 4/5: topic supply profile + candidate inventory policy (all optional). */
211
+ topicProfile?: TopicProfileConfig;
181
212
  /**
182
213
  * Ranking mode (only used when mode='ranking')
183
214
  * - 'day': Daily ranking
@@ -240,6 +240,41 @@ function validateConfig(config, location, databasePath) {
240
240
  if (target.noMatchPolicy?.notify !== undefined && typeof target.noMatchPolicy.notify !== 'boolean') {
241
241
  errors.push(`targets[${index}].noMatchPolicy.notify: Must be a boolean`);
242
242
  }
243
+ if (target.topicProfile) {
244
+ const tp = target.topicProfile;
245
+ if (tp.primary !== undefined && (!Array.isArray(tp.primary) || tp.primary.length === 0 || tp.primary.some((x) => typeof x !== 'string' || !x.trim()))) {
246
+ errors.push(`targets[${index}].topicProfile.primary: Must be a non-empty string array`);
247
+ }
248
+ if (tp.related !== undefined && (!Array.isArray(tp.related) || tp.related.some((x) => typeof x !== 'string' || !x.trim()))) {
249
+ errors.push(`targets[${index}].topicProfile.related: Must be a string array`);
250
+ }
251
+ const st = tp.strategy;
252
+ if (st) {
253
+ for (const field of ['freshnessWeight', 'popularityWeight']) {
254
+ const value = st[field];
255
+ if (value !== undefined && (typeof value !== 'number' || value < 0 || value > 1)) {
256
+ errors.push(`targets[${index}].topicProfile.strategy.${field}: Must be a number between 0 and 1`);
257
+ }
258
+ }
259
+ }
260
+ const inv = tp.inventory;
261
+ if (inv) {
262
+ if (inv.enabled !== undefined && typeof inv.enabled !== 'boolean') {
263
+ errors.push(`targets[${index}].topicProfile.inventory.enabled: Must be a boolean`);
264
+ }
265
+ const bound = (field) => {
266
+ const value = inv[field];
267
+ if (value !== undefined && (typeof value !== 'number' || !Number.isInteger(value) || value < 1 || value > 365)) {
268
+ errors.push(`targets[${index}].topicProfile.inventory.${field}: Must be an integer between 1 and 365`);
269
+ }
270
+ };
271
+ bound('maxAgeDays');
272
+ bound('reserveSize');
273
+ if (inv.fallback !== undefined && typeof inv.fallback !== 'boolean') {
274
+ errors.push(`targets[${index}].topicProfile.inventory.fallback: Must be a boolean`);
275
+ }
276
+ }
277
+ }
243
278
  if (target.noMatchPolicy?.notify === true) {
244
279
  const deliveryTarget = target.delivery?.target?.trim();
245
280
  const notifyTarget = deliveryTarget ? config.delivery?.targets?.[deliveryTarget] : undefined;
@@ -50,6 +50,7 @@ export interface ScheduleOutcomePayload {
50
50
  workType: string;
51
51
  status: string;
52
52
  workId?: string | null;
53
+ candidateReport?: Record<string, unknown> | null;
53
54
  }>;
54
55
  }
55
56
  export declare class DeliveryService {
@@ -176,6 +176,7 @@ class HttpMultipartDelivery {
176
176
  stage: t.stage ?? null,
177
177
  retryable: t.retryable ?? null,
178
178
  operator_hint: t.operator_hint ?? null,
179
+ candidate_report: t.candidate_report ?? null,
179
180
  })),
180
181
  }
181
182
  : { text: request.text, idempotency_key: request.idempotencyKey };
@@ -146,6 +146,8 @@ export interface DeliveryNotificationRequest {
146
146
  /** Whether a later/manual attempt may succeed; terminal does not mean auto-retry pending. */
147
147
  retryable?: boolean | null;
148
148
  operator_hint?: string | null;
149
+ /** Phase 1 Candidate Supply Report ({fetched, selected, rejected, reasons}). */
150
+ candidate_report?: Record<string, unknown> | null;
149
151
  }>;
150
152
  };
151
153
  }
@@ -25,6 +25,11 @@ export declare class IllustrationTargetHandler {
25
25
  * explicit verdict instead of an ambiguous "completed".
26
26
  */
27
27
  private scan;
28
+ /**
29
+ * Upstream topic-supply funnel accumulated across a lookback scan (Phase 1
30
+ * Candidate Report: fetched/selected/rejected + reasons).
31
+ */
32
+ private supplyRep;
28
33
  /**
29
34
  * Global candidate-scan bound (`download.candidateScanLimit`), supplied by
30
35
  * DownloadManager. This is what lets the FETCH stage ask for more than one
@@ -63,6 +68,14 @@ export declare class IllustrationTargetHandler {
63
68
  private fetchIllustrations;
64
69
  private fetchTopicIllustrations;
65
70
  private handleTopicWithLookback;
71
+ /**
72
+ * Phase 5 fallback: claim oldest pending candidate(s) from the durable
73
+ * 待发池 and try to publish them exactly like a normal day's pipeline run.
74
+ * Returns true when at least one work reached a delivery/stored outcome.
75
+ */
76
+ private tryInventoryFallback;
77
+ /** Mark inventory rows whose work reached a delivery/stored outcome. */
78
+ private markInventorySubmittedFromOutcomes;
66
79
  private resolveTopicDay;
67
80
  private shiftDay;
68
81
  private fetchRankingIllustrations;
@@ -12,6 +12,7 @@ const observability_1 = require("../../observability");
12
12
  const context_1 = require("../../observability/context");
13
13
  const DownloadPlanner_1 = require("../plan/DownloadPlanner");
14
14
  const deliveryContext_1 = require("./deliveryContext");
15
+ const inventory_1 = require("../inventory");
15
16
  class IllustrationTargetHandler {
16
17
  client;
17
18
  database;
@@ -29,6 +30,11 @@ class IllustrationTargetHandler {
29
30
  * explicit verdict instead of an ambiguous "completed".
30
31
  */
31
32
  scan = null;
33
+ /**
34
+ * Upstream topic-supply funnel accumulated across a lookback scan (Phase 1
35
+ * Candidate Report: fetched/selected/rejected + reasons).
36
+ */
37
+ supplyRep = null;
32
38
  /**
33
39
  * Global candidate-scan bound (`download.candidateScanLimit`), supplied by
34
40
  * DownloadManager. This is what lets the FETCH stage ask for more than one
@@ -68,6 +74,7 @@ class IllustrationTargetHandler {
68
74
  async handle(target, execution) {
69
75
  this.outcomes = [];
70
76
  this.scan = null;
77
+ this.supplyRep = null;
71
78
  this.execution = execution && (0, WorkIdentity_1.isSingleWorkCell)(target) ? execution : null;
72
79
  // A cell that already owns a work is in RECOVERY, not in a new selection.
73
80
  // Crash/shutdown recovery is not an intentional second run: running the
@@ -113,6 +120,9 @@ class IllustrationTargetHandler {
113
120
  * scheduled slot report success after submitting nothing.
114
121
  */
115
122
  summarize(target) {
123
+ if (this.scan && this.supplyRep) {
124
+ this.scan = { ...this.scan, supply: (0, TargetOutcome_1.withScanSkips)(this.supplyRep, this.scan) };
125
+ }
116
126
  const scan = this.scan ?? undefined;
117
127
  const submitted = this.outcomes.find((o) => o.kind === 'submitted');
118
128
  if (submitted)
@@ -198,7 +208,8 @@ class IllustrationTargetHandler {
198
208
  }
199
209
  async fetchIllustrations(target, mode) {
200
210
  if (mode === 'topic') {
201
- return this.fetchTopicIllustrations(target);
211
+ const { works } = await this.fetchTopicIllustrations(target);
212
+ return works;
202
213
  }
203
214
  if (mode === 'ranking') {
204
215
  return this.fetchRankingIllustrations(target);
@@ -223,7 +234,22 @@ class IllustrationTargetHandler {
223
234
  const pipeline = this.topicPipelineFactory();
224
235
  const { works, selection } = await pipeline.selectWorks(target, 'illustration', day, selectionLimit, target.topicDiscovery ?? {}, target.candidateCollection ?? {});
225
236
  logger_1.logger.info(`Topic "${topic}" illustration: tags=${selection.resolvedTagCount} raw=${selection.rawCount} deduped=${selection.dedupedCount} aiExcluded=${selection.aiExcludedCount} accepted=${selection.acceptedCount} candidates=${works.length} target=${limit}`);
226
- return works;
237
+ // Phase 5: harvest the accepted-but-not-yet-delivered works into the
238
+ // durable 待发池 (idempotent; only when the target opts in).
239
+ (0, inventory_1.recordInventoryCandidates)(this.database, target, topic, day, works, 'illustration');
240
+ // Candidate Supply Observability: fold this lookback day's upstream funnel
241
+ // into the target's accumulated Candidate Report.
242
+ const report = (0, TargetOutcome_1.emptyCandidateSupplyReport)();
243
+ report.fetched = selection.rawCount;
244
+ report.selected = selection.acceptedCount;
245
+ report.rejected = Math.max(0, selection.rawCount - selection.acceptedCount);
246
+ report.reasons = [
247
+ { code: 'duplicate', count: selection.duplicateRemovedCount },
248
+ { code: 'ai_filtered', count: selection.aiExcludedCount },
249
+ { code: 'metadata_filtered', count: Math.max(0, selection.dedupedCount - selection.acceptedCount) },
250
+ ].filter((r) => r.count > 0);
251
+ this.supplyRep = (0, TargetOutcome_1.mergeCandidateSupplyReports)(this.supplyRep ?? undefined, report);
252
+ return { works, selection };
227
253
  }
228
254
  async handleTopicWithLookback(target, displayTag) {
229
255
  const requested = target.limit || 1;
@@ -241,13 +267,14 @@ class IllustrationTargetHandler {
241
267
  fallbackOffset: offset,
242
268
  });
243
269
  }
244
- const illusts = await this.fetchTopicIllustrations(attemptTarget);
270
+ const { works: illusts } = await this.fetchTopicIllustrations(attemptTarget);
245
271
  const result = await this.pipeline.run(illusts, attemptTarget, 'illustration', (illust, tag) => this.downloadAndDeliver(illust, tag, attemptTarget));
246
272
  // The lookback loop is one bounded scan: accumulate every day's
247
273
  // skips/outages so the final verdict covers all candidates attempted.
248
274
  this.scan = (0, TargetOutcome_1.mergeScanSummaries)(this.scan, result.scan);
249
275
  if (result.downloaded > 0) {
250
276
  this.handleDownloadResult(result, target, 'topic', illusts.length);
277
+ this.markInventorySubmittedFromOutcomes(target, (0, inventory_1.inventoryTopic)(target), 'illustration');
251
278
  return;
252
279
  }
253
280
  if (this.scan && this.scan.outages.length > 0) {
@@ -256,6 +283,16 @@ class IllustrationTargetHandler {
256
283
  return;
257
284
  }
258
285
  }
286
+ // Phase 5: when the fresh + lookback scan produced nothing, and the target
287
+ // opted into CandidateInventory, publish oldest pending reserve candidates.
288
+ const topic = (0, inventory_1.inventoryTopic)(target);
289
+ if (await this.tryInventoryFallback(target, topic, 'illustration')) {
290
+ return;
291
+ }
292
+ // Report how much reserve remains so the empty result is not a dead end.
293
+ if (this.supplyRep) {
294
+ this.supplyRep = (0, inventory_1.attachInventoryReport)(this.database, target, topic, this.supplyRep);
295
+ }
259
296
  const scan = this.scan;
260
297
  // An explicit no-eligible-candidate verdict needs something to have been
261
298
  // considered. When the scan is empty (nothing surfaced at all) the richer
@@ -275,6 +312,65 @@ class IllustrationTargetHandler {
275
312
  ...(scan ? { scan } : {}),
276
313
  });
277
314
  }
315
+ /**
316
+ * Phase 5 fallback: claim oldest pending candidate(s) from the durable
317
+ * 待发池 and try to publish them exactly like a normal day's pipeline run.
318
+ * Returns true when at least one work reached a delivery/stored outcome.
319
+ */
320
+ async tryInventoryFallback(target, topic, workType) {
321
+ const policy = (0, inventory_1.inventoryPolicy)(target);
322
+ if (!policy.enabled || policy.fallback === false)
323
+ return false;
324
+ const targetId = (0, inventory_1.inventoryTargetId)(target);
325
+ const repo = this.database.candidateInventory;
326
+ while (true) {
327
+ const row = repo.claimNext({ topic, targetId, reserveSize: policy.reserveSize, date: (0, pixiv_date_utils_1.getTodayDate)() });
328
+ if (!row)
329
+ break;
330
+ const id = Number(row.pixivId);
331
+ if (!Number.isFinite(id)) {
332
+ repo.markFiltered(row.pixivId, row.workType, topic, targetId);
333
+ continue;
334
+ }
335
+ let detail;
336
+ try {
337
+ detail = await this.client.getIllustration(id);
338
+ }
339
+ catch (error) {
340
+ this.logError(error, `Inventory candidate illustration ${id} could not be fetched`);
341
+ repo.markSelectedBackToPending(row.pixivId, row.workType, topic, targetId);
342
+ continue;
343
+ }
344
+ const attemptTarget = { ...target };
345
+ const result = await this.pipeline.run([detail], attemptTarget, 'illustration', (illust, tag) => this.downloadAndDeliver(illust, tag, attemptTarget));
346
+ this.scan = (0, TargetOutcome_1.mergeScanSummaries)(this.scan, result.scan);
347
+ if (result.downloaded > 0) {
348
+ this.handleDownloadResult(result, attemptTarget, 'topic', 1);
349
+ (0, inventory_1.markInventoryAttempt)(this.database, target, topic, row.pixivId, 'illustration', 'submitted');
350
+ return true;
351
+ }
352
+ if (this.scan && this.scan.outages.length > 0) {
353
+ // A dead token / dead network is not "no candidate": retry the same
354
+ // row on the next run instead of discarding it.
355
+ (0, inventory_1.markInventoryAttempt)(this.database, target, topic, row.pixivId, 'illustration', 'pending');
356
+ return false;
357
+ }
358
+ // The claimed work is no longer usable (deleted/private/duplicate): drop
359
+ // it permanently and try the next pending row.
360
+ (0, inventory_1.markInventoryAttempt)(this.database, target, topic, row.pixivId, 'illustration', 'filtered');
361
+ }
362
+ return false;
363
+ }
364
+ /** Mark inventory rows whose work reached a delivery/stored outcome. */
365
+ markInventorySubmittedFromOutcomes(target, topic, workType) {
366
+ if (!(0, inventory_1.inventoryPolicy)(target).enabled)
367
+ return;
368
+ for (const outcome of this.outcomes) {
369
+ if (outcome.kind === 'delivery_pending' || outcome.kind === 'submitted' || outcome.kind === 'stored') {
370
+ (0, inventory_1.markInventoryAttempt)(this.database, target, topic, outcome.workId, workType, 'submitted');
371
+ }
372
+ }
373
+ }
278
374
  resolveTopicDay(target) {
279
375
  return target.date === 'TODAY'
280
376
  ? (0, pixiv_date_utils_1.getTodayDate)()
@@ -24,6 +24,10 @@ export declare class NovelTargetHandler {
24
24
  * explicit verdict instead of an ambiguous "completed".
25
25
  */
26
26
  private scan;
27
+ /**
28
+ * Upstream topic-supply funnel accumulated across a lookback scan.
29
+ */
30
+ private supplyRep;
27
31
  /**
28
32
  * Global candidate-scan bound (`download.candidateScanLimit`), supplied by
29
33
  * DownloadManager. This is what lets the FETCH stage ask for more than one
@@ -63,6 +67,13 @@ export declare class NovelTargetHandler {
63
67
  private fetchNovels;
64
68
  private fetchTopicNovels;
65
69
  private handleTopicWithLookback;
70
+ /**
71
+ * Phase 5 fallback: claim pending novel candidates and publish them into the
72
+ * existing aggregate day-scan (same idempotent pipeline as a normal day).
73
+ */
74
+ private tryInventoryFallback;
75
+ /** Mark inventory rows whose work reached a delivery/stored outcome. */
76
+ private markInventorySubmittedFromOutcomes;
66
77
  private resolveTopicDay;
67
78
  private shiftDay;
68
79
  private fetchRankingNovels;
@@ -12,6 +12,7 @@ const observability_1 = require("../../observability");
12
12
  const context_1 = require("../../observability/context");
13
13
  const DownloadPlanner_1 = require("../plan/DownloadPlanner");
14
14
  const deliveryContext_1 = require("./deliveryContext");
15
+ const inventory_1 = require("../inventory");
15
16
  const TelePressRichNovel_1 = require("../../delivery/TelePressRichNovel");
16
17
  class NovelTargetHandler {
17
18
  client;
@@ -29,6 +30,10 @@ class NovelTargetHandler {
29
30
  * explicit verdict instead of an ambiguous "completed".
30
31
  */
31
32
  scan = null;
33
+ /**
34
+ * Upstream topic-supply funnel accumulated across a lookback scan.
35
+ */
36
+ supplyRep = null;
32
37
  /**
33
38
  * Global candidate-scan bound (`download.candidateScanLimit`), supplied by
34
39
  * DownloadManager. This is what lets the FETCH stage ask for more than one
@@ -68,6 +73,7 @@ class NovelTargetHandler {
68
73
  async handle(target, execution) {
69
74
  this.outcomes = [];
70
75
  this.scan = null;
76
+ this.supplyRep = null;
71
77
  this.execution = execution && (0, WorkIdentity_1.isSingleWorkCell)(target) ? execution : null;
72
78
  // A cell that already owns a work is in RECOVERY, not in a new selection.
73
79
  // Crash/shutdown recovery is not an intentional second run: running the
@@ -126,6 +132,9 @@ class NovelTargetHandler {
126
132
  * single-work RECOVERY path, whose cell identity is fixed.
127
133
  */
128
134
  summarize() {
135
+ if (this.scan && this.supplyRep) {
136
+ this.scan = { ...this.scan, supply: (0, TargetOutcome_1.withScanSkips)(this.supplyRep, this.scan) };
137
+ }
129
138
  const scan = this.scan ?? undefined;
130
139
  const submitted = this.outcomes.find((o) => o.kind === 'submitted');
131
140
  if (submitted)
@@ -200,7 +209,8 @@ class NovelTargetHandler {
200
209
  }
201
210
  async fetchNovels(target, mode) {
202
211
  if (mode === 'topic') {
203
- return this.fetchTopicNovels(target);
212
+ const { works } = await this.fetchTopicNovels(target);
213
+ return works;
204
214
  }
205
215
  if (mode === 'ranking') {
206
216
  return this.fetchRankingNovels(target);
@@ -223,7 +233,17 @@ class NovelTargetHandler {
223
233
  const pipeline = this.topicPipelineFactory();
224
234
  const { works, selection } = await pipeline.selectWorks(target, 'novel', day, selectionLimit, target.topicDiscovery ?? {}, target.candidateCollection ?? {});
225
235
  logger_1.logger.info(`Topic "${topic}" novel: tags=${selection.resolvedTagCount} raw=${selection.rawCount} deduped=${selection.dedupedCount} accepted=${selection.acceptedCount} candidates=${works.length} target=${limit}`);
226
- return works;
236
+ (0, inventory_1.recordInventoryCandidates)(this.database, target, topic, day, works, 'novel');
237
+ const report = (0, TargetOutcome_1.emptyCandidateSupplyReport)();
238
+ report.fetched = selection.rawCount;
239
+ report.selected = selection.acceptedCount;
240
+ report.rejected = Math.max(0, selection.rawCount - selection.acceptedCount);
241
+ report.reasons = [
242
+ { code: 'duplicate', count: selection.duplicateRemovedCount },
243
+ { code: 'metadata_filtered', count: Math.max(0, selection.dedupedCount - selection.acceptedCount) },
244
+ ].filter((r) => r.count > 0);
245
+ this.supplyRep = (0, TargetOutcome_1.mergeCandidateSupplyReports)(this.supplyRep ?? undefined, report);
246
+ return { works, selection };
227
247
  }
228
248
  async handleTopicWithLookback(target, displayTag) {
229
249
  const requested = target.limit || 1;
@@ -253,7 +273,7 @@ class NovelTargetHandler {
253
273
  fallbackOffset: offset,
254
274
  });
255
275
  }
256
- const novels = await this.fetchTopicNovels(attemptTarget);
276
+ const { works: novels } = await this.fetchTopicNovels(attemptTarget);
257
277
  totalFound += novels.length;
258
278
  const result = await this.pipeline.run(novels, attemptTarget, 'novel', (novel, tag) => this.downloadAndDeliver(novel, tag, attemptTarget));
259
279
  // The lookback loop is ONE bounded scan: accumulate every day's
@@ -263,15 +283,84 @@ class NovelTargetHandler {
263
283
  aggregate.skipped += result.skipped;
264
284
  aggregate.alreadyDownloaded += result.alreadyDownloaded;
265
285
  aggregate.filteredOut += result.filteredOut;
286
+ if (result.downloaded > 0) {
287
+ this.markInventorySubmittedFromOutcomes(target, (0, inventory_1.inventoryTopic)(target), 'novel');
288
+ }
266
289
  if (aggregate.scan.outages.length > 0) {
267
290
  // A dead token / dead database / dead network is not "no matching
268
291
  // novel": stop looking back and let the job fail/retry.
269
292
  break;
270
293
  }
271
294
  }
295
+ const topic = (0, inventory_1.inventoryTopic)(target);
296
+ await this.tryInventoryFallback(target, topic, aggregate);
297
+ if (this.supplyRep) {
298
+ this.supplyRep = (0, inventory_1.attachInventoryReport)(this.database, target, topic, this.supplyRep);
299
+ }
272
300
  this.scan = aggregate.scan;
273
301
  await this.handleDownloadResult(aggregate, target, 'topic', totalFound, checkedDays);
274
302
  }
303
+ /**
304
+ * Phase 5 fallback: claim pending novel candidates and publish them into the
305
+ * existing aggregate day-scan (same idempotent pipeline as a normal day).
306
+ */
307
+ async tryInventoryFallback(target, topic, aggregate) {
308
+ const policy = (0, inventory_1.inventoryPolicy)(target);
309
+ if (!policy.enabled || policy.fallback === false)
310
+ return;
311
+ // Only a fully-empty fresh/lookback window may draw from the reserve; a
312
+ // partial day stays on fresh supply rather than padding beyond the limit.
313
+ if (aggregate.downloaded > 0)
314
+ return;
315
+ const targetId = (0, inventory_1.inventoryTargetId)(target);
316
+ const repo = this.database.candidateInventory;
317
+ while (aggregate.downloaded < (target.limit || 1)) {
318
+ const row = repo.claimNext({ topic, targetId, reserveSize: policy.reserveSize, date: (0, pixiv_date_utils_1.getTodayDate)() });
319
+ if (!row)
320
+ break;
321
+ const id = Number(row.pixivId);
322
+ if (!Number.isFinite(id)) {
323
+ repo.markFiltered(row.pixivId, row.workType, topic, targetId);
324
+ continue;
325
+ }
326
+ let detail;
327
+ try {
328
+ detail = await this.client.getNovelDetail(id);
329
+ }
330
+ catch (error) {
331
+ this.logError(error, `Inventory candidate novel ${id} could not be fetched`);
332
+ repo.markSelectedBackToPending(row.pixivId, row.workType, topic, targetId);
333
+ continue;
334
+ }
335
+ const attemptTarget = {
336
+ ...target,
337
+ limit: (target.limit || 1) - aggregate.downloaded,
338
+ };
339
+ const result = await this.pipeline.run([detail], attemptTarget, 'novel', (novel, tag) => this.downloadAndDeliver(novel, tag, attemptTarget));
340
+ aggregate.scan = (0, TargetOutcome_1.mergeScanSummaries)(aggregate.scan, result.scan);
341
+ aggregate.downloaded += result.downloaded;
342
+ aggregate.skipped += result.skipped;
343
+ aggregate.alreadyDownloaded += result.alreadyDownloaded;
344
+ aggregate.filteredOut += result.filteredOut;
345
+ if (result.downloaded > 0) {
346
+ (0, inventory_1.markInventoryAttempt)(this.database, target, topic, row.pixivId, 'novel', 'submitted');
347
+ continue;
348
+ }
349
+ if (aggregate.scan.outages.length > 0)
350
+ return;
351
+ (0, inventory_1.markInventoryAttempt)(this.database, target, topic, row.pixivId, 'novel', 'filtered');
352
+ }
353
+ }
354
+ /** Mark inventory rows whose work reached a delivery/stored outcome. */
355
+ markInventorySubmittedFromOutcomes(target, topic, workType) {
356
+ if (!(0, inventory_1.inventoryPolicy)(target).enabled)
357
+ return;
358
+ for (const outcome of this.outcomes) {
359
+ if (outcome.kind === 'delivery_pending' || outcome.kind === 'submitted' || outcome.kind === 'stored') {
360
+ (0, inventory_1.markInventoryAttempt)(this.database, target, topic, outcome.workId, workType, 'submitted');
361
+ }
362
+ }
363
+ }
275
364
  resolveTopicDay(target) {
276
365
  return target.date === 'TODAY'
277
366
  ? (0, pixiv_date_utils_1.getTodayDate)()
@@ -0,0 +1,29 @@
1
+ import type { TargetConfig } from '../config';
2
+ import type { IDatabase } from '../interfaces/IDatabase';
3
+ import type { CandidateSupplyReport } from '../scheduler/TargetOutcome';
4
+ /** Defaults and bounds for the optional Phase 5 inventory. */
5
+ export interface InventoryPolicy {
6
+ enabled: boolean;
7
+ maxAgeDays: number;
8
+ reserveSize: number;
9
+ fallback: boolean;
10
+ }
11
+ export declare function inventoryPolicy(target: TargetConfig): InventoryPolicy;
12
+ export declare function inventoryTopic(target: TargetConfig): string;
13
+ export declare function inventoryTargetId(target: TargetConfig): string;
14
+ /**
15
+ * Phase 5: after a topic scan yields eligible works that were NOT delivered by
16
+ * this run's lookback loop, upsert them as idle pending inventory. Works that
17
+ * are already recorded as downloaded/delivered are skipped by the caller
18
+ * upstream (selectWorks already removed download-history rows); this method is
19
+ * deliberately idempotent.
20
+ */
21
+ export declare function recordInventoryCandidates(database: IDatabase, target: TargetConfig, topic: string, date: string, works: Array<{
22
+ id: number;
23
+ type?: string;
24
+ }>, workType: 'illustration' | 'novel'): void;
25
+ /** Enrich a candidate report with the durable pending-count snapshot. */
26
+ export declare function attachInventoryReport(database: IDatabase, target: TargetConfig, topic: string, report: CandidateSupplyReport): CandidateSupplyReport;
27
+ /** Marks a claimed inventory row after a delivery attempt. */
28
+ export declare function markInventoryAttempt(database: IDatabase, target: TargetConfig, topic: string, workId: string, workType: 'illustration' | 'novel', outcome: 'submitted' | 'filtered' | 'pending'): void;
29
+ //# sourceMappingURL=inventory.d.ts.map
@@ -0,0 +1,80 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.inventoryPolicy = inventoryPolicy;
4
+ exports.inventoryTopic = inventoryTopic;
5
+ exports.inventoryTargetId = inventoryTargetId;
6
+ exports.recordInventoryCandidates = recordInventoryCandidates;
7
+ exports.attachInventoryReport = attachInventoryReport;
8
+ exports.markInventoryAttempt = markInventoryAttempt;
9
+ function inventoryPolicy(target) {
10
+ const cfg = target.topicProfile?.inventory;
11
+ return {
12
+ enabled: cfg?.enabled === true,
13
+ maxAgeDays: cfg?.maxAgeDays ?? 30,
14
+ reserveSize: Math.min(Math.max(cfg?.reserveSize ?? 20, 1), 100),
15
+ fallback: cfg?.fallback ?? true,
16
+ };
17
+ }
18
+ function inventoryTopic(target) {
19
+ return (target.topicProfile?.primary?.[0] ?? target.topic ?? '').trim();
20
+ }
21
+ function inventoryTargetId(target) {
22
+ return target.id ?? inventoryTopic(target);
23
+ }
24
+ /**
25
+ * Phase 5: after a topic scan yields eligible works that were NOT delivered by
26
+ * this run's lookback loop, upsert them as idle pending inventory. Works that
27
+ * are already recorded as downloaded/delivered are skipped by the caller
28
+ * upstream (selectWorks already removed download-history rows); this method is
29
+ * deliberately idempotent.
30
+ */
31
+ function recordInventoryCandidates(database, target, topic, date, works, workType) {
32
+ const policy = inventoryPolicy(target);
33
+ if (!policy.enabled)
34
+ return;
35
+ const targetId = inventoryTargetId(target);
36
+ for (const work of works) {
37
+ const id = String(work?.id);
38
+ if (!id)
39
+ continue;
40
+ database.candidateInventory.upsert({
41
+ pixivId: id,
42
+ workType,
43
+ topic,
44
+ targetId,
45
+ snapshot: work,
46
+ date,
47
+ maxAgeDays: policy.maxAgeDays,
48
+ });
49
+ }
50
+ }
51
+ /** Enrich a candidate report with the durable pending-count snapshot. */
52
+ function attachInventoryReport(database, target, topic, report) {
53
+ const policy = inventoryPolicy(target);
54
+ if (!policy.enabled)
55
+ return report;
56
+ const summary = database.candidateInventory.pendingSummary(topic, inventoryTargetId(target));
57
+ return {
58
+ ...report,
59
+ inventory: {
60
+ pendingCount: summary.count,
61
+ reserveSize: policy.reserveSize,
62
+ maxAgeDays: policy.maxAgeDays,
63
+ oldestSeenDate: summary.oldestSeenDate,
64
+ },
65
+ };
66
+ }
67
+ /** Marks a claimed inventory row after a delivery attempt. */
68
+ function markInventoryAttempt(database, target, topic, workId, workType, outcome) {
69
+ if (!inventoryPolicy(target).enabled)
70
+ return;
71
+ const targetId = inventoryTargetId(target);
72
+ const repo = database.candidateInventory;
73
+ if (outcome === 'submitted')
74
+ repo.markSubmitted(workId, workType, topic, targetId);
75
+ else if (outcome === 'filtered')
76
+ repo.markFiltered(workId, workType, topic, targetId);
77
+ else
78
+ repo.markSelectedBackToPending(workId, workType, topic, targetId);
79
+ }
80
+ //# sourceMappingURL=inventory.js.map
@@ -1,4 +1,5 @@
1
1
  import { AccessTokenStore, DownloadRecordInput, ExecutionStatus, SchedulerExecutionRecord } from '../storage/Database';
2
+ import { CandidateInventoryRepository } from '../storage/repositories/CandidateInventoryRepository';
2
3
  /**
3
4
  * Interface for database operations
4
5
  * Provides abstraction for data persistence
@@ -123,5 +124,7 @@ export interface IDatabase {
123
124
  };
124
125
  countSince(botId: string | null, hours: number): number;
125
126
  };
127
+ /** Phase 5 durable CandidateInventory (待发池). */
128
+ readonly candidateInventory: CandidateInventoryRepository;
126
129
  }
127
130
  //# sourceMappingURL=IDatabase.d.ts.map
@@ -46,6 +46,7 @@ export declare class NotificationPolicy {
46
46
  error: string | null;
47
47
  terminal_reason_code?: string | null;
48
48
  reason?: string | null;
49
+ candidateReport?: Record<string, unknown> | null;
49
50
  }>): void;
50
51
  /**
51
52
  * Delivery targets whose HTTP target declares the given outcome URL.
@@ -145,6 +145,7 @@ class NotificationPolicy {
145
145
  stage: operational?.stage ?? null,
146
146
  retryable: operational?.retryable ?? null,
147
147
  operator_hint: operational?.operatorHint ?? null,
148
+ candidate_report: r.candidateReport ?? null,
148
149
  };
149
150
  }),
150
151
  });
package/dist/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "type": "commonjs",
3
3
  "name": "pixivflow",
4
- "version": "2.28.1",
4
+ "version": "2.30.0",
5
5
  "private": true
6
6
  }