pixivflow 2.29.0 → 2.31.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/README.en.md +2 -2
  2. package/README.md +2 -2
  3. package/dist/commands/WebUICommand.d.ts +2 -2
  4. package/dist/commands/WebUICommand.js +9 -7
  5. package/dist/config/types.d.ts +31 -0
  6. package/dist/config/validation.js +35 -0
  7. package/dist/download/handlers/IllustrationTargetHandler.d.ts +8 -0
  8. package/dist/download/handlers/IllustrationTargetHandler.js +74 -0
  9. package/dist/download/handlers/NovelTargetHandler.d.ts +7 -0
  10. package/dist/download/handlers/NovelTargetHandler.js +71 -0
  11. package/dist/download/inventory.d.ts +29 -0
  12. package/dist/download/inventory.js +80 -0
  13. package/dist/interfaces/IDatabase.d.ts +3 -0
  14. package/dist/package.json +1 -1
  15. package/dist/scheduler/TargetOutcome.d.ts +13 -0
  16. package/dist/scheduler/TargetOutcome.js +3 -1
  17. package/dist/storage/Database.d.ts +4 -0
  18. package/dist/storage/Database.js +7 -0
  19. package/dist/storage/DatabaseMigration.js +20 -0
  20. package/dist/storage/repositories/CandidateInventoryRepository.d.ts +51 -0
  21. package/dist/storage/repositories/CandidateInventoryRepository.js +129 -0
  22. package/dist/version.js +1 -1
  23. package/dist/webui/package.json +1 -1
  24. package/package.json +2 -1
  25. package/webui-frontend/dist/assets/CheckCircleOutlined-H3DnUyMO.js +2 -0
  26. package/webui-frontend/dist/assets/ClearOutlined-BpODq7I7.js +2 -0
  27. package/webui-frontend/dist/assets/CloseCircleOutlined-BFfS5iXd.js +2 -0
  28. package/webui-frontend/dist/assets/Config-DRElCeDC.js +16 -0
  29. package/webui-frontend/dist/assets/Dashboard-aqPMquSO.js +2 -0
  30. package/webui-frontend/dist/assets/DeleteOutlined-BFmW1oWe.js +2 -0
  31. package/webui-frontend/dist/assets/Download-DCKDU51K.js +2 -0
  32. package/webui-frontend/dist/assets/Files-nNoRY5tl.js +5 -0
  33. package/webui-frontend/dist/assets/PictureOutlined-U51jGr2U.js +2 -0
  34. package/webui-frontend/dist/assets/SortDescendingOutlined-CckVuL_b.js +2 -0
  35. package/webui-frontend/dist/assets/ThunderboltOutlined-ulcCd7ui.js +2 -0
  36. package/webui-frontend/dist/assets/UrlDownload-Bzs0AZN0.js +3 -0
  37. package/webui-frontend/dist/assets/dateUtils-gQt2uAjS.js +2 -0
  38. package/webui-frontend/dist/assets/errorCodeTranslator-BqIackMm.js +3 -0
  39. package/webui-frontend/dist/assets/index-B2ns27ii.js +2 -0
  40. package/webui-frontend/dist/assets/index-B7yYDv10.js +2 -0
  41. package/webui-frontend/dist/assets/index-Bbsa44oT.js +2 -0
  42. package/webui-frontend/dist/assets/index-C2ANILg9.js +19 -0
  43. package/webui-frontend/dist/assets/index-CDrp3NIf.js +8 -0
  44. package/webui-frontend/dist/assets/index-CFtP5lqo.js +22 -0
  45. package/webui-frontend/dist/assets/index-CJln3zA7.js +14 -0
  46. package/webui-frontend/dist/assets/index-CZhfhXvc.js +2 -0
  47. package/webui-frontend/dist/assets/index-CfHs6AUn.js +2 -0
  48. package/webui-frontend/dist/assets/index-DG6kGbYq.js +217 -0
  49. package/webui-frontend/dist/assets/index-DHwJxrFi.css +1 -0
  50. package/webui-frontend/dist/assets/index-DPigBPWX.js +2 -0
  51. package/webui-frontend/dist/assets/index-DarObP-C.js +2 -0
  52. package/webui-frontend/dist/assets/index-_EG4kaKN.js +47 -0
  53. package/webui-frontend/dist/assets/row-_c51T55h.js +20 -0
  54. package/webui-frontend/dist/assets/socket-oqYz9cKb.js +2 -0
  55. package/webui-frontend/dist/assets/useAuth-wCjAOx6S.js +9 -0
  56. package/webui-frontend/dist/assets/useConfig-Ds7RpIL1.js +9 -0
  57. package/webui-frontend/dist/assets/useDownload-BlxvZNEm.js +2 -0
  58. package/webui-frontend/dist/assets/useErrorHandler-BOvHue1w.js +84 -0
  59. package/webui-frontend/dist/index.html +15 -0
package/README.en.md CHANGED
@@ -69,7 +69,7 @@ pixivflow scheduler # long-running cron collection
69
69
  ```
70
70
 
71
71
  Prefer not to hand-write config? The interactive wizard `pixivflow setup` generates it.
72
- For a GUI, run `pixivflow webui` (frontend:
72
+ For a GUI, run `pixivflow web` (frontend:
73
73
  [pixivflow-webui](https://github.com/redtidev1918/pixivflow-webui)).
74
74
 
75
75
  Downloading Pixiv ugoira additionally needs `python3` and `ffmpeg`: frames are
@@ -249,7 +249,7 @@ ledger; dry-run by default).
249
249
  | `pixivflow download --url <url>` | Direct download via URL |
250
250
  | `pixivflow random` | Random popular artwork |
251
251
  | `pixivflow scheduler` | Start scheduled jobs |
252
- | `pixivflow webui` | Start the WebUI |
252
+ | `pixivflow web` | Start the WebUI |
253
253
  | `pixivflow config` | Manage config (view / edit / backup / restore) |
254
254
  | `pixivflow status` | Download stats and recent records |
255
255
  | `pixivflow health` | Health check: config, directories, connectivity |
package/README.md CHANGED
@@ -64,7 +64,7 @@ pixivflow scheduler # 按 cron 配置长期挂机自动收集
64
64
  ```
65
65
 
66
66
  不想手写配置?运行交互式向导 `pixivflow setup` 一步步生成。图形界面用
67
- `pixivflow webui`(前端见 [pixivflow-webui](https://github.com/redtidev1918/pixivflow-webui))。
67
+ `pixivflow web`(前端见 [pixivflow-webui](https://github.com/redtidev1918/pixivflow-webui))。
68
68
 
69
69
  下载 Pixiv 动图(ugoira)还需要 `python3` 和 `ffmpeg`:程序按逐帧延迟合成循环 GIF,
70
70
  可直接作为动画交付给下游。官方 Docker 镜像已包含两者,详见
@@ -227,7 +227,7 @@ pixivflow outbox cancel <id> # 只取消尚未执行的 row
227
227
  | `pixivflow download --url <url>` | 通过 URL 直接下载 |
228
228
  | `pixivflow random` | 随机下载热门作品 |
229
229
  | `pixivflow scheduler` | 启动定时任务 |
230
- | `pixivflow webui` | 启动 WebUI |
230
+ | `pixivflow web` | 启动 WebUI |
231
231
  | `pixivflow config` | 配置管理(查看 / 编辑 / 备份 / 恢复) |
232
232
  | `pixivflow status` | 下载统计与最近记录 |
233
233
  | `pixivflow health` | 健康检查:配置、目录可写性、连通性 |
@@ -8,8 +8,8 @@ import { CommandContext, CommandArgs, CommandResult } from './types';
8
8
  * WebUI command - Start WebUI server
9
9
  */
10
10
  export declare class WebUICommand extends BaseCommand {
11
- readonly name = "webui";
12
- readonly description = "Start WebUI server";
11
+ readonly name = "web";
12
+ readonly description = "Start WebUI server (memory only when used)";
13
13
  readonly aliases: string[];
14
14
  readonly metadata: {
15
15
  category: CommandCategory;
@@ -18,9 +18,9 @@ const child_process_1 = require("child_process");
18
18
  * WebUI command - Start WebUI server
19
19
  */
20
20
  class WebUICommand extends Command_1.BaseCommand {
21
- name = 'webui';
22
- description = 'Start WebUI server';
23
- aliases = ['w'];
21
+ name = 'web';
22
+ description = 'Start WebUI server (memory only when used)';
23
+ aliases = ['w', 'webui'];
24
24
  metadata = {
25
25
  category: metadata_1.CommandCategory.UTILITY,
26
26
  requiresAuth: false,
@@ -437,10 +437,12 @@ Auto-Build Feature:
437
437
  the command will automatically build the frontend before starting.
438
438
 
439
439
  Examples:
440
- pixivflow webui # Auto-build if in source directory
441
- pixivflow webui --port 8080 # Use custom port
442
- pixivflow webui --host 0.0.0.0 # Bind to all interfaces
443
- pixivflow webui --static-path ./webui-frontend/dist # Use custom static path`;
440
+ pixivflow web # Start WebUI (loads frontend only now)
441
+ pixivflow web --port 8080 # Use custom port
442
+ pixivflow web --host 0.0.0.0 # Bind to all interfaces
443
+ pixivflow web --static-path ./webui-frontend/dist # Use custom static path
444
+
445
+ Backward-compatible aliases: pixivflow webui, pixivflow w`;
444
446
  }
445
447
  }
446
448
  exports.WebUICommand = WebUICommand;
@@ -27,6 +27,35 @@ export interface CandidateCollectionConfig {
27
27
  /** Minimum metadata-topic score to survive filtering (default 0.35). */
28
28
  minMetadataScore?: number;
29
29
  }
30
+ /** Phase 4: topic-supply strategy (per-target, optional, default off). */
31
+ export interface TopicProfileStrategyConfig {
32
+ /** Relative weight for publishing fresher works first (0..1, default 0.4). */
33
+ freshnessWeight?: number;
34
+ /** Relative weight for publishing higher-popularity works first (0..1, default 0.3). */
35
+ popularityWeight?: number;
36
+ }
37
+ /** Phase 5: durable candidate inventory ("待发池") for a target. */
38
+ export interface CandidateInventoryConfig {
39
+ /** Master switch. Default false: no collection, no fallback, no new columns read. */
40
+ enabled?: boolean;
41
+ /** Max age (days) a pending candidate may stay usable for fallback. Default 30. */
42
+ maxAgeDays?: number;
43
+ /** Max pending rows retained per (topic,target). Default 20. */
44
+ reserveSize?: number;
45
+ /** When the fresh/lookback scan is empty, publish oldest pending inventory. Default true. */
46
+ fallback?: boolean;
47
+ }
48
+ /** Phase 4: TopicProfile (seed + related tags + supply strategy). */
49
+ export interface TopicProfileConfig {
50
+ /** Primary seed tag(s); first item is the canonical topic when topic is unset. */
51
+ primary?: string[];
52
+ /** Related tags that strengthen topic membership evidence. */
53
+ related?: string[];
54
+ /** Ranking strategy knobs (Phase 4; wired after CandidateInventory). */
55
+ strategy?: TopicProfileStrategyConfig;
56
+ /** CandidateInventory policy (Phase 5). */
57
+ inventory?: CandidateInventoryConfig;
58
+ }
30
59
  /** Behaviour when a target cannot produce the requested number of works. */
31
60
  export interface NoMatchPolicyConfig {
32
61
  /**
@@ -178,6 +207,8 @@ export interface TargetConfig {
178
207
  candidateCollection?: CandidateCollectionConfig;
179
208
  /** Bounded fallback and notification policy for an empty result. */
180
209
  noMatchPolicy?: NoMatchPolicyConfig;
210
+ /** Phase 4/5: topic supply profile + candidate inventory policy (all optional). */
211
+ topicProfile?: TopicProfileConfig;
181
212
  /**
182
213
  * Ranking mode (only used when mode='ranking')
183
214
  * - 'day': Daily ranking
@@ -240,6 +240,41 @@ function validateConfig(config, location, databasePath) {
240
240
  if (target.noMatchPolicy?.notify !== undefined && typeof target.noMatchPolicy.notify !== 'boolean') {
241
241
  errors.push(`targets[${index}].noMatchPolicy.notify: Must be a boolean`);
242
242
  }
243
+ if (target.topicProfile) {
244
+ const tp = target.topicProfile;
245
+ if (tp.primary !== undefined && (!Array.isArray(tp.primary) || tp.primary.length === 0 || tp.primary.some((x) => typeof x !== 'string' || !x.trim()))) {
246
+ errors.push(`targets[${index}].topicProfile.primary: Must be a non-empty string array`);
247
+ }
248
+ if (tp.related !== undefined && (!Array.isArray(tp.related) || tp.related.some((x) => typeof x !== 'string' || !x.trim()))) {
249
+ errors.push(`targets[${index}].topicProfile.related: Must be a string array`);
250
+ }
251
+ const st = tp.strategy;
252
+ if (st) {
253
+ for (const field of ['freshnessWeight', 'popularityWeight']) {
254
+ const value = st[field];
255
+ if (value !== undefined && (typeof value !== 'number' || value < 0 || value > 1)) {
256
+ errors.push(`targets[${index}].topicProfile.strategy.${field}: Must be a number between 0 and 1`);
257
+ }
258
+ }
259
+ }
260
+ const inv = tp.inventory;
261
+ if (inv) {
262
+ if (inv.enabled !== undefined && typeof inv.enabled !== 'boolean') {
263
+ errors.push(`targets[${index}].topicProfile.inventory.enabled: Must be a boolean`);
264
+ }
265
+ const bound = (field) => {
266
+ const value = inv[field];
267
+ if (value !== undefined && (typeof value !== 'number' || !Number.isInteger(value) || value < 1 || value > 365)) {
268
+ errors.push(`targets[${index}].topicProfile.inventory.${field}: Must be an integer between 1 and 365`);
269
+ }
270
+ };
271
+ bound('maxAgeDays');
272
+ bound('reserveSize');
273
+ if (inv.fallback !== undefined && typeof inv.fallback !== 'boolean') {
274
+ errors.push(`targets[${index}].topicProfile.inventory.fallback: Must be a boolean`);
275
+ }
276
+ }
277
+ }
243
278
  if (target.noMatchPolicy?.notify === true) {
244
279
  const deliveryTarget = target.delivery?.target?.trim();
245
280
  const notifyTarget = deliveryTarget ? config.delivery?.targets?.[deliveryTarget] : undefined;
@@ -68,6 +68,14 @@ export declare class IllustrationTargetHandler {
68
68
  private fetchIllustrations;
69
69
  private fetchTopicIllustrations;
70
70
  private handleTopicWithLookback;
71
+ /**
72
+ * Phase 5 fallback: claim oldest pending candidate(s) from the durable
73
+ * 待发池 and try to publish them exactly like a normal day's pipeline run.
74
+ * Returns true when at least one work reached a delivery/stored outcome.
75
+ */
76
+ private tryInventoryFallback;
77
+ /** Mark inventory rows whose work reached a delivery/stored outcome. */
78
+ private markInventorySubmittedFromOutcomes;
71
79
  private resolveTopicDay;
72
80
  private shiftDay;
73
81
  private fetchRankingIllustrations;
@@ -12,6 +12,7 @@ const observability_1 = require("../../observability");
12
12
  const context_1 = require("../../observability/context");
13
13
  const DownloadPlanner_1 = require("../plan/DownloadPlanner");
14
14
  const deliveryContext_1 = require("./deliveryContext");
15
+ const inventory_1 = require("../inventory");
15
16
  class IllustrationTargetHandler {
16
17
  client;
17
18
  database;
@@ -233,6 +234,9 @@ class IllustrationTargetHandler {
233
234
  const pipeline = this.topicPipelineFactory();
234
235
  const { works, selection } = await pipeline.selectWorks(target, 'illustration', day, selectionLimit, target.topicDiscovery ?? {}, target.candidateCollection ?? {});
235
236
  logger_1.logger.info(`Topic "${topic}" illustration: tags=${selection.resolvedTagCount} raw=${selection.rawCount} deduped=${selection.dedupedCount} aiExcluded=${selection.aiExcludedCount} accepted=${selection.acceptedCount} candidates=${works.length} target=${limit}`);
237
+ // Phase 5: harvest the accepted-but-not-yet-delivered works into the
238
+ // durable 待发池 (idempotent; only when the target opts in).
239
+ (0, inventory_1.recordInventoryCandidates)(this.database, target, topic, day, works, 'illustration');
236
240
  // Candidate Supply Observability: fold this lookback day's upstream funnel
237
241
  // into the target's accumulated Candidate Report.
238
242
  const report = (0, TargetOutcome_1.emptyCandidateSupplyReport)();
@@ -270,6 +274,7 @@ class IllustrationTargetHandler {
270
274
  this.scan = (0, TargetOutcome_1.mergeScanSummaries)(this.scan, result.scan);
271
275
  if (result.downloaded > 0) {
272
276
  this.handleDownloadResult(result, target, 'topic', illusts.length);
277
+ this.markInventorySubmittedFromOutcomes(target, (0, inventory_1.inventoryTopic)(target), 'illustration');
273
278
  return;
274
279
  }
275
280
  if (this.scan && this.scan.outages.length > 0) {
@@ -278,6 +283,16 @@ class IllustrationTargetHandler {
278
283
  return;
279
284
  }
280
285
  }
286
+ // Phase 5: when the fresh + lookback scan produced nothing, and the target
287
+ // opted into CandidateInventory, publish oldest pending reserve candidates.
288
+ const topic = (0, inventory_1.inventoryTopic)(target);
289
+ if (await this.tryInventoryFallback(target, topic, 'illustration')) {
290
+ return;
291
+ }
292
+ // Report how much reserve remains so the empty result is not a dead end.
293
+ if (this.supplyRep) {
294
+ this.supplyRep = (0, inventory_1.attachInventoryReport)(this.database, target, topic, this.supplyRep);
295
+ }
281
296
  const scan = this.scan;
282
297
  // An explicit no-eligible-candidate verdict needs something to have been
283
298
  // considered. When the scan is empty (nothing surfaced at all) the richer
@@ -297,6 +312,65 @@ class IllustrationTargetHandler {
297
312
  ...(scan ? { scan } : {}),
298
313
  });
299
314
  }
315
+ /**
316
+ * Phase 5 fallback: claim oldest pending candidate(s) from the durable
317
+ * 待发池 and try to publish them exactly like a normal day's pipeline run.
318
+ * Returns true when at least one work reached a delivery/stored outcome.
319
+ */
320
+ async tryInventoryFallback(target, topic, workType) {
321
+ const policy = (0, inventory_1.inventoryPolicy)(target);
322
+ if (!policy.enabled || policy.fallback === false)
323
+ return false;
324
+ const targetId = (0, inventory_1.inventoryTargetId)(target);
325
+ const repo = this.database.candidateInventory;
326
+ while (true) {
327
+ const row = repo.claimNext({ topic, targetId, reserveSize: policy.reserveSize, date: (0, pixiv_date_utils_1.getTodayDate)() });
328
+ if (!row)
329
+ break;
330
+ const id = Number(row.pixivId);
331
+ if (!Number.isFinite(id)) {
332
+ repo.markFiltered(row.pixivId, row.workType, topic, targetId);
333
+ continue;
334
+ }
335
+ let detail;
336
+ try {
337
+ detail = await this.client.getIllustration(id);
338
+ }
339
+ catch (error) {
340
+ this.logError(error, `Inventory candidate illustration ${id} could not be fetched`);
341
+ repo.markSelectedBackToPending(row.pixivId, row.workType, topic, targetId);
342
+ continue;
343
+ }
344
+ const attemptTarget = { ...target };
345
+ const result = await this.pipeline.run([detail], attemptTarget, 'illustration', (illust, tag) => this.downloadAndDeliver(illust, tag, attemptTarget));
346
+ this.scan = (0, TargetOutcome_1.mergeScanSummaries)(this.scan, result.scan);
347
+ if (result.downloaded > 0) {
348
+ this.handleDownloadResult(result, attemptTarget, 'topic', 1);
349
+ (0, inventory_1.markInventoryAttempt)(this.database, target, topic, row.pixivId, 'illustration', 'submitted');
350
+ return true;
351
+ }
352
+ if (this.scan && this.scan.outages.length > 0) {
353
+ // A dead token / dead network is not "no candidate": retry the same
354
+ // row on the next run instead of discarding it.
355
+ (0, inventory_1.markInventoryAttempt)(this.database, target, topic, row.pixivId, 'illustration', 'pending');
356
+ return false;
357
+ }
358
+ // The claimed work is no longer usable (deleted/private/duplicate): drop
359
+ // it permanently and try the next pending row.
360
+ (0, inventory_1.markInventoryAttempt)(this.database, target, topic, row.pixivId, 'illustration', 'filtered');
361
+ }
362
+ return false;
363
+ }
364
+ /** Mark inventory rows whose work reached a delivery/stored outcome. */
365
+ markInventorySubmittedFromOutcomes(target, topic, workType) {
366
+ if (!(0, inventory_1.inventoryPolicy)(target).enabled)
367
+ return;
368
+ for (const outcome of this.outcomes) {
369
+ if (outcome.kind === 'delivery_pending' || outcome.kind === 'submitted' || outcome.kind === 'stored') {
370
+ (0, inventory_1.markInventoryAttempt)(this.database, target, topic, outcome.workId, workType, 'submitted');
371
+ }
372
+ }
373
+ }
300
374
  resolveTopicDay(target) {
301
375
  return target.date === 'TODAY'
302
376
  ? (0, pixiv_date_utils_1.getTodayDate)()
@@ -67,6 +67,13 @@ export declare class NovelTargetHandler {
67
67
  private fetchNovels;
68
68
  private fetchTopicNovels;
69
69
  private handleTopicWithLookback;
70
+ /**
71
+ * Phase 5 fallback: claim pending novel candidates and publish them into the
72
+ * existing aggregate day-scan (same idempotent pipeline as a normal day).
73
+ */
74
+ private tryInventoryFallback;
75
+ /** Mark inventory rows whose work reached a delivery/stored outcome. */
76
+ private markInventorySubmittedFromOutcomes;
70
77
  private resolveTopicDay;
71
78
  private shiftDay;
72
79
  private fetchRankingNovels;
@@ -12,6 +12,7 @@ const observability_1 = require("../../observability");
12
12
  const context_1 = require("../../observability/context");
13
13
  const DownloadPlanner_1 = require("../plan/DownloadPlanner");
14
14
  const deliveryContext_1 = require("./deliveryContext");
15
+ const inventory_1 = require("../inventory");
15
16
  const TelePressRichNovel_1 = require("../../delivery/TelePressRichNovel");
16
17
  class NovelTargetHandler {
17
18
  client;
@@ -232,6 +233,7 @@ class NovelTargetHandler {
232
233
  const pipeline = this.topicPipelineFactory();
233
234
  const { works, selection } = await pipeline.selectWorks(target, 'novel', day, selectionLimit, target.topicDiscovery ?? {}, target.candidateCollection ?? {});
234
235
  logger_1.logger.info(`Topic "${topic}" novel: tags=${selection.resolvedTagCount} raw=${selection.rawCount} deduped=${selection.dedupedCount} accepted=${selection.acceptedCount} candidates=${works.length} target=${limit}`);
236
+ (0, inventory_1.recordInventoryCandidates)(this.database, target, topic, day, works, 'novel');
235
237
  const report = (0, TargetOutcome_1.emptyCandidateSupplyReport)();
236
238
  report.fetched = selection.rawCount;
237
239
  report.selected = selection.acceptedCount;
@@ -281,15 +283,84 @@ class NovelTargetHandler {
281
283
  aggregate.skipped += result.skipped;
282
284
  aggregate.alreadyDownloaded += result.alreadyDownloaded;
283
285
  aggregate.filteredOut += result.filteredOut;
286
+ if (result.downloaded > 0) {
287
+ this.markInventorySubmittedFromOutcomes(target, (0, inventory_1.inventoryTopic)(target), 'novel');
288
+ }
284
289
  if (aggregate.scan.outages.length > 0) {
285
290
  // A dead token / dead database / dead network is not "no matching
286
291
  // novel": stop looking back and let the job fail/retry.
287
292
  break;
288
293
  }
289
294
  }
295
+ const topic = (0, inventory_1.inventoryTopic)(target);
296
+ await this.tryInventoryFallback(target, topic, aggregate);
297
+ if (this.supplyRep) {
298
+ this.supplyRep = (0, inventory_1.attachInventoryReport)(this.database, target, topic, this.supplyRep);
299
+ }
290
300
  this.scan = aggregate.scan;
291
301
  await this.handleDownloadResult(aggregate, target, 'topic', totalFound, checkedDays);
292
302
  }
303
+ /**
304
+ * Phase 5 fallback: claim pending novel candidates and publish them into the
305
+ * existing aggregate day-scan (same idempotent pipeline as a normal day).
306
+ */
307
+ async tryInventoryFallback(target, topic, aggregate) {
308
+ const policy = (0, inventory_1.inventoryPolicy)(target);
309
+ if (!policy.enabled || policy.fallback === false)
310
+ return;
311
+ // Only a fully-empty fresh/lookback window may draw from the reserve; a
312
+ // partial day stays on fresh supply rather than padding beyond the limit.
313
+ if (aggregate.downloaded > 0)
314
+ return;
315
+ const targetId = (0, inventory_1.inventoryTargetId)(target);
316
+ const repo = this.database.candidateInventory;
317
+ while (aggregate.downloaded < (target.limit || 1)) {
318
+ const row = repo.claimNext({ topic, targetId, reserveSize: policy.reserveSize, date: (0, pixiv_date_utils_1.getTodayDate)() });
319
+ if (!row)
320
+ break;
321
+ const id = Number(row.pixivId);
322
+ if (!Number.isFinite(id)) {
323
+ repo.markFiltered(row.pixivId, row.workType, topic, targetId);
324
+ continue;
325
+ }
326
+ let detail;
327
+ try {
328
+ detail = await this.client.getNovelDetail(id);
329
+ }
330
+ catch (error) {
331
+ this.logError(error, `Inventory candidate novel ${id} could not be fetched`);
332
+ repo.markSelectedBackToPending(row.pixivId, row.workType, topic, targetId);
333
+ continue;
334
+ }
335
+ const attemptTarget = {
336
+ ...target,
337
+ limit: (target.limit || 1) - aggregate.downloaded,
338
+ };
339
+ const result = await this.pipeline.run([detail], attemptTarget, 'novel', (novel, tag) => this.downloadAndDeliver(novel, tag, attemptTarget));
340
+ aggregate.scan = (0, TargetOutcome_1.mergeScanSummaries)(aggregate.scan, result.scan);
341
+ aggregate.downloaded += result.downloaded;
342
+ aggregate.skipped += result.skipped;
343
+ aggregate.alreadyDownloaded += result.alreadyDownloaded;
344
+ aggregate.filteredOut += result.filteredOut;
345
+ if (result.downloaded > 0) {
346
+ (0, inventory_1.markInventoryAttempt)(this.database, target, topic, row.pixivId, 'novel', 'submitted');
347
+ continue;
348
+ }
349
+ if (aggregate.scan.outages.length > 0)
350
+ return;
351
+ (0, inventory_1.markInventoryAttempt)(this.database, target, topic, row.pixivId, 'novel', 'filtered');
352
+ }
353
+ }
354
+ /** Mark inventory rows whose work reached a delivery/stored outcome. */
355
+ markInventorySubmittedFromOutcomes(target, topic, workType) {
356
+ if (!(0, inventory_1.inventoryPolicy)(target).enabled)
357
+ return;
358
+ for (const outcome of this.outcomes) {
359
+ if (outcome.kind === 'delivery_pending' || outcome.kind === 'submitted' || outcome.kind === 'stored') {
360
+ (0, inventory_1.markInventoryAttempt)(this.database, target, topic, outcome.workId, workType, 'submitted');
361
+ }
362
+ }
363
+ }
293
364
  resolveTopicDay(target) {
294
365
  return target.date === 'TODAY'
295
366
  ? (0, pixiv_date_utils_1.getTodayDate)()
@@ -0,0 +1,29 @@
1
+ import type { TargetConfig } from '../config';
2
+ import type { IDatabase } from '../interfaces/IDatabase';
3
+ import type { CandidateSupplyReport } from '../scheduler/TargetOutcome';
4
+ /** Defaults and bounds for the optional Phase 5 inventory. */
5
+ export interface InventoryPolicy {
6
+ enabled: boolean;
7
+ maxAgeDays: number;
8
+ reserveSize: number;
9
+ fallback: boolean;
10
+ }
11
+ export declare function inventoryPolicy(target: TargetConfig): InventoryPolicy;
12
+ export declare function inventoryTopic(target: TargetConfig): string;
13
+ export declare function inventoryTargetId(target: TargetConfig): string;
14
+ /**
15
+ * Phase 5: after a topic scan yields eligible works that were NOT delivered by
16
+ * this run's lookback loop, upsert them as idle pending inventory. Works that
17
+ * are already recorded as downloaded/delivered are skipped by the caller
18
+ * upstream (selectWorks already removed download-history rows); this method is
19
+ * deliberately idempotent.
20
+ */
21
+ export declare function recordInventoryCandidates(database: IDatabase, target: TargetConfig, topic: string, date: string, works: Array<{
22
+ id: number;
23
+ type?: string;
24
+ }>, workType: 'illustration' | 'novel'): void;
25
+ /** Enrich a candidate report with the durable pending-count snapshot. */
26
+ export declare function attachInventoryReport(database: IDatabase, target: TargetConfig, topic: string, report: CandidateSupplyReport): CandidateSupplyReport;
27
+ /** Marks a claimed inventory row after a delivery attempt. */
28
+ export declare function markInventoryAttempt(database: IDatabase, target: TargetConfig, topic: string, workId: string, workType: 'illustration' | 'novel', outcome: 'submitted' | 'filtered' | 'pending'): void;
29
+ //# sourceMappingURL=inventory.d.ts.map
@@ -0,0 +1,80 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.inventoryPolicy = inventoryPolicy;
4
+ exports.inventoryTopic = inventoryTopic;
5
+ exports.inventoryTargetId = inventoryTargetId;
6
+ exports.recordInventoryCandidates = recordInventoryCandidates;
7
+ exports.attachInventoryReport = attachInventoryReport;
8
+ exports.markInventoryAttempt = markInventoryAttempt;
9
+ function inventoryPolicy(target) {
10
+ const cfg = target.topicProfile?.inventory;
11
+ return {
12
+ enabled: cfg?.enabled === true,
13
+ maxAgeDays: cfg?.maxAgeDays ?? 30,
14
+ reserveSize: Math.min(Math.max(cfg?.reserveSize ?? 20, 1), 100),
15
+ fallback: cfg?.fallback ?? true,
16
+ };
17
+ }
18
+ function inventoryTopic(target) {
19
+ return (target.topicProfile?.primary?.[0] ?? target.topic ?? '').trim();
20
+ }
21
+ function inventoryTargetId(target) {
22
+ return target.id ?? inventoryTopic(target);
23
+ }
24
+ /**
25
+ * Phase 5: after a topic scan yields eligible works that were NOT delivered by
26
+ * this run's lookback loop, upsert them as idle pending inventory. Works that
27
+ * are already recorded as downloaded/delivered are skipped by the caller
28
+ * upstream (selectWorks already removed download-history rows); this method is
29
+ * deliberately idempotent.
30
+ */
31
+ function recordInventoryCandidates(database, target, topic, date, works, workType) {
32
+ const policy = inventoryPolicy(target);
33
+ if (!policy.enabled)
34
+ return;
35
+ const targetId = inventoryTargetId(target);
36
+ for (const work of works) {
37
+ const id = String(work?.id);
38
+ if (!id)
39
+ continue;
40
+ database.candidateInventory.upsert({
41
+ pixivId: id,
42
+ workType,
43
+ topic,
44
+ targetId,
45
+ snapshot: work,
46
+ date,
47
+ maxAgeDays: policy.maxAgeDays,
48
+ });
49
+ }
50
+ }
51
+ /** Enrich a candidate report with the durable pending-count snapshot. */
52
+ function attachInventoryReport(database, target, topic, report) {
53
+ const policy = inventoryPolicy(target);
54
+ if (!policy.enabled)
55
+ return report;
56
+ const summary = database.candidateInventory.pendingSummary(topic, inventoryTargetId(target));
57
+ return {
58
+ ...report,
59
+ inventory: {
60
+ pendingCount: summary.count,
61
+ reserveSize: policy.reserveSize,
62
+ maxAgeDays: policy.maxAgeDays,
63
+ oldestSeenDate: summary.oldestSeenDate,
64
+ },
65
+ };
66
+ }
67
+ /** Marks a claimed inventory row after a delivery attempt. */
68
+ function markInventoryAttempt(database, target, topic, workId, workType, outcome) {
69
+ if (!inventoryPolicy(target).enabled)
70
+ return;
71
+ const targetId = inventoryTargetId(target);
72
+ const repo = database.candidateInventory;
73
+ if (outcome === 'submitted')
74
+ repo.markSubmitted(workId, workType, topic, targetId);
75
+ else if (outcome === 'filtered')
76
+ repo.markFiltered(workId, workType, topic, targetId);
77
+ else
78
+ repo.markSelectedBackToPending(workId, workType, topic, targetId);
79
+ }
80
+ //# sourceMappingURL=inventory.js.map
@@ -1,4 +1,5 @@
1
1
  import { AccessTokenStore, DownloadRecordInput, ExecutionStatus, SchedulerExecutionRecord } from '../storage/Database';
2
+ import { CandidateInventoryRepository } from '../storage/repositories/CandidateInventoryRepository';
2
3
  /**
3
4
  * Interface for database operations
4
5
  * Provides abstraction for data persistence
@@ -123,5 +124,7 @@ export interface IDatabase {
123
124
  };
124
125
  countSince(botId: string | null, hours: number): number;
125
126
  };
127
+ /** Phase 5 durable CandidateInventory (待发池). */
128
+ readonly candidateInventory: CandidateInventoryRepository;
126
129
  }
127
130
  //# sourceMappingURL=IDatabase.d.ts.map
package/dist/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "type": "commonjs",
3
3
  "name": "pixivflow",
4
- "version": "2.29.0",
4
+ "version": "2.31.0",
5
5
  "private": true
6
6
  }
@@ -109,6 +109,17 @@ export interface CandidateSupplyReason {
109
109
  code: string;
110
110
  count: number;
111
111
  }
112
+ /** Phase 5: durable 待发池 snapshot for sparse topics (only when enabled). */
113
+ export interface CandidateInventoryReport {
114
+ /** Rows still usable for fallback at report time. */
115
+ pendingCount: number;
116
+ /** Target's configured max retained rows. */
117
+ reserveSize: number;
118
+ /** Max age in days before a pending candidate expires. */
119
+ maxAgeDays: number;
120
+ /** Earliest still-pending first-seen date (YYYY-MM-DD), null when empty. */
121
+ oldestSeenDate: string | null;
122
+ }
112
123
  /** Candidate-supply observability snapshot (Phase 1 Candidate Report). */
113
124
  export interface CandidateSupplyReport {
114
125
  /** Total works surfaced by the topic search before any filtering. */
@@ -119,6 +130,8 @@ export interface CandidateSupplyReport {
119
130
  rejected: number;
120
131
  /** Why candidates were rejected, by reason code (extensible). */
121
132
  reasons: CandidateSupplyReason[];
133
+ /** Phase 5: durable 待发池 reserve (optional, present only when enabled). */
134
+ inventory?: CandidateInventoryReport;
122
135
  }
123
136
  /**
124
137
  * A freshly-harvested upstream funnel with no candidates selected yet.
@@ -20,7 +20,7 @@ exports.terminalReasonFor = terminalReasonFor;
20
20
  * A freshly-harvested upstream funnel with no candidates selected yet.
21
21
  */
22
22
  function emptyCandidateSupplyReport() {
23
- return { fetched: 0, selected: 0, rejected: 0, reasons: [] };
23
+ return { fetched: 0, selected: 0, rejected: 0, reasons: [], inventory: undefined };
24
24
  }
25
25
  /**
26
26
  * Fold two supply snapshots of the SAME logical target together, so a
@@ -43,6 +43,7 @@ function mergeCandidateSupplyReports(first, second) {
43
43
  selected: first.selected + second.selected,
44
44
  rejected: Math.max(0, first.fetched + second.fetched - (first.selected + second.selected)),
45
45
  reasons,
46
+ inventory: second.inventory ?? first.inventory,
46
47
  };
47
48
  }
48
49
  /**
@@ -68,6 +69,7 @@ function withScanSkips(report, scan) {
68
69
  selected,
69
70
  rejected: Math.max(0, base.fetched - selected),
70
71
  reasons,
72
+ inventory: base.inventory,
71
73
  };
72
74
  }
73
75
  /**
@@ -5,6 +5,7 @@ import { OutboxRepository } from './repositories/OutboxRepository';
5
5
  import { MetadataRepository } from './repositories/MetadataRepository';
6
6
  import { SQLiteRateLimitStateStore } from './repositories/RateLimitStateRepository';
7
7
  import { SystemErrorRepository } from './repositories/SystemErrorRepository';
8
+ import { CandidateInventoryRepository } from './repositories/CandidateInventoryRepository';
8
9
  export interface AccessTokenStore {
9
10
  accessToken: string;
10
11
  expiresAt: number;
@@ -48,6 +49,7 @@ export declare class Database implements IDatabase {
48
49
  private metadataRepo;
49
50
  private rateLimitStateStore;
50
51
  private systemErrorRepo;
52
+ private candidateInventoryRepo;
51
53
  constructor(databasePath: string);
52
54
  migrate(): void;
53
55
  /** Absolute path of the SQLite file (used to locate sibling cache dirs). */
@@ -64,6 +66,8 @@ export declare class Database implements IDatabase {
64
66
  get rateLimitState(): SQLiteRateLimitStateStore;
65
67
  /** Durable system-error ledger (observability; never throws). */
66
68
  get systemErrors(): SystemErrorRepository;
69
+ /** Phase 5 CandidateInventory (待发池) for sparse topics. */
70
+ get candidateInventory(): CandidateInventoryRepository;
67
71
  /** Raw transactional boundary for atomic multi-table intents. */
68
72
  transaction<T>(fn: () => T): T;
69
73
  /** Expose a prepared-statement helper if needed by services (pragmas etc). */
@@ -18,6 +18,7 @@ const OutboxRepository_1 = require("./repositories/OutboxRepository");
18
18
  const MetadataRepository_1 = require("./repositories/MetadataRepository");
19
19
  const RateLimitStateRepository_1 = require("./repositories/RateLimitStateRepository");
20
20
  const SystemErrorRepository_1 = require("./repositories/SystemErrorRepository");
21
+ const CandidateInventoryRepository_1 = require("./repositories/CandidateInventoryRepository");
21
22
  const NodeSqliteDriver_1 = require("./drivers/NodeSqliteDriver");
22
23
  class Database {
23
24
  databasePath;
@@ -35,6 +36,7 @@ class Database {
35
36
  metadataRepo;
36
37
  rateLimitStateStore;
37
38
  systemErrorRepo;
39
+ candidateInventoryRepo;
38
40
  constructor(databasePath) {
39
41
  this.databasePath = databasePath;
40
42
  try {
@@ -63,6 +65,7 @@ class Database {
63
65
  this.metadataRepo = new MetadataRepository_1.MetadataRepository(this.db);
64
66
  this.rateLimitStateStore = new RateLimitStateRepository_1.SQLiteRateLimitStateStore(this.db);
65
67
  this.systemErrorRepo = new SystemErrorRepository_1.SystemErrorRepository(this.db);
68
+ this.candidateInventoryRepo = new CandidateInventoryRepository_1.CandidateInventoryRepository(this.db);
66
69
  }
67
70
  catch (error) {
68
71
  throw new errors_1.DatabaseError(`Failed to initialize database at ${this.databasePath}`, error instanceof Error ? error : undefined);
@@ -102,6 +105,10 @@ class Database {
102
105
  get systemErrors() {
103
106
  return this.systemErrorRepo;
104
107
  }
108
+ /** Phase 5 CandidateInventory (待发池) for sparse topics. */
109
+ get candidateInventory() {
110
+ return this.candidateInventoryRepo;
111
+ }
105
112
  /** Raw transactional boundary for atomic multi-table intents. */
106
113
  transaction(fn) {
107
114
  return this.db.transaction(fn)();