pixivflow 2.29.0 → 2.30.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -27,6 +27,35 @@ export interface CandidateCollectionConfig {
27
27
  /** Minimum metadata-topic score to survive filtering (default 0.35). */
28
28
  minMetadataScore?: number;
29
29
  }
30
+ /** Phase 4: topic-supply strategy (per-target, optional, default off). */
31
+ export interface TopicProfileStrategyConfig {
32
+ /** Relative weight for publishing fresher works first (0..1, default 0.4). */
33
+ freshnessWeight?: number;
34
+ /** Relative weight for publishing higher-popularity works first (0..1, default 0.3). */
35
+ popularityWeight?: number;
36
+ }
37
+ /** Phase 5: durable candidate inventory ("待发池") for a target. */
38
+ export interface CandidateInventoryConfig {
39
+ /** Master switch. Default false: no collection, no fallback, no new columns read. */
40
+ enabled?: boolean;
41
+ /** Max age (days) a pending candidate may stay usable for fallback. Default 30. */
42
+ maxAgeDays?: number;
43
+ /** Max pending rows retained per (topic,target). Default 20. */
44
+ reserveSize?: number;
45
+ /** When the fresh/lookback scan is empty, publish oldest pending inventory. Default true. */
46
+ fallback?: boolean;
47
+ }
48
+ /** Phase 4: TopicProfile (seed + related tags + supply strategy). */
49
+ export interface TopicProfileConfig {
50
+ /** Primary seed tag(s); first item is the canonical topic when topic is unset. */
51
+ primary?: string[];
52
+ /** Related tags that strengthen topic membership evidence. */
53
+ related?: string[];
54
+ /** Ranking strategy knobs (Phase 4; wired after CandidateInventory). */
55
+ strategy?: TopicProfileStrategyConfig;
56
+ /** CandidateInventory policy (Phase 5). */
57
+ inventory?: CandidateInventoryConfig;
58
+ }
30
59
  /** Behaviour when a target cannot produce the requested number of works. */
31
60
  export interface NoMatchPolicyConfig {
32
61
  /**
@@ -178,6 +207,8 @@ export interface TargetConfig {
178
207
  candidateCollection?: CandidateCollectionConfig;
179
208
  /** Bounded fallback and notification policy for an empty result. */
180
209
  noMatchPolicy?: NoMatchPolicyConfig;
210
+ /** Phase 4/5: topic supply profile + candidate inventory policy (all optional). */
211
+ topicProfile?: TopicProfileConfig;
181
212
  /**
182
213
  * Ranking mode (only used when mode='ranking')
183
214
  * - 'day': Daily ranking
@@ -240,6 +240,41 @@ function validateConfig(config, location, databasePath) {
240
240
  if (target.noMatchPolicy?.notify !== undefined && typeof target.noMatchPolicy.notify !== 'boolean') {
241
241
  errors.push(`targets[${index}].noMatchPolicy.notify: Must be a boolean`);
242
242
  }
243
+ if (target.topicProfile) {
244
+ const tp = target.topicProfile;
245
+ if (tp.primary !== undefined && (!Array.isArray(tp.primary) || tp.primary.length === 0 || tp.primary.some((x) => typeof x !== 'string' || !x.trim()))) {
246
+ errors.push(`targets[${index}].topicProfile.primary: Must be a non-empty string array`);
247
+ }
248
+ if (tp.related !== undefined && (!Array.isArray(tp.related) || tp.related.some((x) => typeof x !== 'string' || !x.trim()))) {
249
+ errors.push(`targets[${index}].topicProfile.related: Must be a string array`);
250
+ }
251
+ const st = tp.strategy;
252
+ if (st) {
253
+ for (const field of ['freshnessWeight', 'popularityWeight']) {
254
+ const value = st[field];
255
+ if (value !== undefined && (typeof value !== 'number' || value < 0 || value > 1)) {
256
+ errors.push(`targets[${index}].topicProfile.strategy.${field}: Must be a number between 0 and 1`);
257
+ }
258
+ }
259
+ }
260
+ const inv = tp.inventory;
261
+ if (inv) {
262
+ if (inv.enabled !== undefined && typeof inv.enabled !== 'boolean') {
263
+ errors.push(`targets[${index}].topicProfile.inventory.enabled: Must be a boolean`);
264
+ }
265
+ const bound = (field) => {
266
+ const value = inv[field];
267
+ if (value !== undefined && (typeof value !== 'number' || !Number.isInteger(value) || value < 1 || value > 365)) {
268
+ errors.push(`targets[${index}].topicProfile.inventory.${field}: Must be an integer between 1 and 365`);
269
+ }
270
+ };
271
+ bound('maxAgeDays');
272
+ bound('reserveSize');
273
+ if (inv.fallback !== undefined && typeof inv.fallback !== 'boolean') {
274
+ errors.push(`targets[${index}].topicProfile.inventory.fallback: Must be a boolean`);
275
+ }
276
+ }
277
+ }
243
278
  if (target.noMatchPolicy?.notify === true) {
244
279
  const deliveryTarget = target.delivery?.target?.trim();
245
280
  const notifyTarget = deliveryTarget ? config.delivery?.targets?.[deliveryTarget] : undefined;
@@ -68,6 +68,14 @@ export declare class IllustrationTargetHandler {
68
68
  private fetchIllustrations;
69
69
  private fetchTopicIllustrations;
70
70
  private handleTopicWithLookback;
71
+ /**
72
+ * Phase 5 fallback: claim oldest pending candidate(s) from the durable
73
+ * 待发池 and try to publish them exactly like a normal day's pipeline run.
74
+ * Returns true when at least one work reached a delivery/stored outcome.
75
+ */
76
+ private tryInventoryFallback;
77
+ /** Mark inventory rows whose work reached a delivery/stored outcome. */
78
+ private markInventorySubmittedFromOutcomes;
71
79
  private resolveTopicDay;
72
80
  private shiftDay;
73
81
  private fetchRankingIllustrations;
@@ -12,6 +12,7 @@ const observability_1 = require("../../observability");
12
12
  const context_1 = require("../../observability/context");
13
13
  const DownloadPlanner_1 = require("../plan/DownloadPlanner");
14
14
  const deliveryContext_1 = require("./deliveryContext");
15
+ const inventory_1 = require("../inventory");
15
16
  class IllustrationTargetHandler {
16
17
  client;
17
18
  database;
@@ -233,6 +234,9 @@ class IllustrationTargetHandler {
233
234
  const pipeline = this.topicPipelineFactory();
234
235
  const { works, selection } = await pipeline.selectWorks(target, 'illustration', day, selectionLimit, target.topicDiscovery ?? {}, target.candidateCollection ?? {});
235
236
  logger_1.logger.info(`Topic "${topic}" illustration: tags=${selection.resolvedTagCount} raw=${selection.rawCount} deduped=${selection.dedupedCount} aiExcluded=${selection.aiExcludedCount} accepted=${selection.acceptedCount} candidates=${works.length} target=${limit}`);
237
+ // Phase 5: harvest the accepted-but-not-yet-delivered works into the
238
+ // durable 待发池 (idempotent; only when the target opts in).
239
+ (0, inventory_1.recordInventoryCandidates)(this.database, target, topic, day, works, 'illustration');
236
240
  // Candidate Supply Observability: fold this lookback day's upstream funnel
237
241
  // into the target's accumulated Candidate Report.
238
242
  const report = (0, TargetOutcome_1.emptyCandidateSupplyReport)();
@@ -270,6 +274,7 @@ class IllustrationTargetHandler {
270
274
  this.scan = (0, TargetOutcome_1.mergeScanSummaries)(this.scan, result.scan);
271
275
  if (result.downloaded > 0) {
272
276
  this.handleDownloadResult(result, target, 'topic', illusts.length);
277
+ this.markInventorySubmittedFromOutcomes(target, (0, inventory_1.inventoryTopic)(target), 'illustration');
273
278
  return;
274
279
  }
275
280
  if (this.scan && this.scan.outages.length > 0) {
@@ -278,6 +283,16 @@ class IllustrationTargetHandler {
278
283
  return;
279
284
  }
280
285
  }
286
+ // Phase 5: when the fresh + lookback scan produced nothing, and the target
287
+ // opted into CandidateInventory, publish oldest pending reserve candidates.
288
+ const topic = (0, inventory_1.inventoryTopic)(target);
289
+ if (await this.tryInventoryFallback(target, topic, 'illustration')) {
290
+ return;
291
+ }
292
+ // Report how much reserve remains so the empty result is not a dead end.
293
+ if (this.supplyRep) {
294
+ this.supplyRep = (0, inventory_1.attachInventoryReport)(this.database, target, topic, this.supplyRep);
295
+ }
281
296
  const scan = this.scan;
282
297
  // An explicit no-eligible-candidate verdict needs something to have been
283
298
  // considered. When the scan is empty (nothing surfaced at all) the richer
@@ -297,6 +312,65 @@ class IllustrationTargetHandler {
297
312
  ...(scan ? { scan } : {}),
298
313
  });
299
314
  }
315
+ /**
316
+ * Phase 5 fallback: claim oldest pending candidate(s) from the durable
317
+ * 待发池 and try to publish them exactly like a normal day's pipeline run.
318
+ * Returns true when at least one work reached a delivery/stored outcome.
319
+ */
320
+ async tryInventoryFallback(target, topic, workType) {
321
+ const policy = (0, inventory_1.inventoryPolicy)(target);
322
+ if (!policy.enabled || policy.fallback === false)
323
+ return false;
324
+ const targetId = (0, inventory_1.inventoryTargetId)(target);
325
+ const repo = this.database.candidateInventory;
326
+ while (true) {
327
+ const row = repo.claimNext({ topic, targetId, reserveSize: policy.reserveSize, date: (0, pixiv_date_utils_1.getTodayDate)() });
328
+ if (!row)
329
+ break;
330
+ const id = Number(row.pixivId);
331
+ if (!Number.isFinite(id)) {
332
+ repo.markFiltered(row.pixivId, row.workType, topic, targetId);
333
+ continue;
334
+ }
335
+ let detail;
336
+ try {
337
+ detail = await this.client.getIllustration(id);
338
+ }
339
+ catch (error) {
340
+ this.logError(error, `Inventory candidate illustration ${id} could not be fetched`);
341
+ repo.markSelectedBackToPending(row.pixivId, row.workType, topic, targetId);
342
+ continue;
343
+ }
344
+ const attemptTarget = { ...target };
345
+ const result = await this.pipeline.run([detail], attemptTarget, 'illustration', (illust, tag) => this.downloadAndDeliver(illust, tag, attemptTarget));
346
+ this.scan = (0, TargetOutcome_1.mergeScanSummaries)(this.scan, result.scan);
347
+ if (result.downloaded > 0) {
348
+ this.handleDownloadResult(result, attemptTarget, 'topic', 1);
349
+ (0, inventory_1.markInventoryAttempt)(this.database, target, topic, row.pixivId, 'illustration', 'submitted');
350
+ return true;
351
+ }
352
+ if (this.scan && this.scan.outages.length > 0) {
353
+ // A dead token / dead network is not "no candidate": retry the same
354
+ // row on the next run instead of discarding it.
355
+ (0, inventory_1.markInventoryAttempt)(this.database, target, topic, row.pixivId, 'illustration', 'pending');
356
+ return false;
357
+ }
358
+ // The claimed work is no longer usable (deleted/private/duplicate): drop
359
+ // it permanently and try the next pending row.
360
+ (0, inventory_1.markInventoryAttempt)(this.database, target, topic, row.pixivId, 'illustration', 'filtered');
361
+ }
362
+ return false;
363
+ }
364
+ /** Mark inventory rows whose work reached a delivery/stored outcome. */
365
+ markInventorySubmittedFromOutcomes(target, topic, workType) {
366
+ if (!(0, inventory_1.inventoryPolicy)(target).enabled)
367
+ return;
368
+ for (const outcome of this.outcomes) {
369
+ if (outcome.kind === 'delivery_pending' || outcome.kind === 'submitted' || outcome.kind === 'stored') {
370
+ (0, inventory_1.markInventoryAttempt)(this.database, target, topic, outcome.workId, workType, 'submitted');
371
+ }
372
+ }
373
+ }
300
374
  resolveTopicDay(target) {
301
375
  return target.date === 'TODAY'
302
376
  ? (0, pixiv_date_utils_1.getTodayDate)()
@@ -67,6 +67,13 @@ export declare class NovelTargetHandler {
67
67
  private fetchNovels;
68
68
  private fetchTopicNovels;
69
69
  private handleTopicWithLookback;
70
+ /**
71
+ * Phase 5 fallback: claim pending novel candidates and publish them into the
72
+ * existing aggregate day-scan (same idempotent pipeline as a normal day).
73
+ */
74
+ private tryInventoryFallback;
75
+ /** Mark inventory rows whose work reached a delivery/stored outcome. */
76
+ private markInventorySubmittedFromOutcomes;
70
77
  private resolveTopicDay;
71
78
  private shiftDay;
72
79
  private fetchRankingNovels;
@@ -12,6 +12,7 @@ const observability_1 = require("../../observability");
12
12
  const context_1 = require("../../observability/context");
13
13
  const DownloadPlanner_1 = require("../plan/DownloadPlanner");
14
14
  const deliveryContext_1 = require("./deliveryContext");
15
+ const inventory_1 = require("../inventory");
15
16
  const TelePressRichNovel_1 = require("../../delivery/TelePressRichNovel");
16
17
  class NovelTargetHandler {
17
18
  client;
@@ -232,6 +233,7 @@ class NovelTargetHandler {
232
233
  const pipeline = this.topicPipelineFactory();
233
234
  const { works, selection } = await pipeline.selectWorks(target, 'novel', day, selectionLimit, target.topicDiscovery ?? {}, target.candidateCollection ?? {});
234
235
  logger_1.logger.info(`Topic "${topic}" novel: tags=${selection.resolvedTagCount} raw=${selection.rawCount} deduped=${selection.dedupedCount} accepted=${selection.acceptedCount} candidates=${works.length} target=${limit}`);
236
+ (0, inventory_1.recordInventoryCandidates)(this.database, target, topic, day, works, 'novel');
235
237
  const report = (0, TargetOutcome_1.emptyCandidateSupplyReport)();
236
238
  report.fetched = selection.rawCount;
237
239
  report.selected = selection.acceptedCount;
@@ -281,15 +283,84 @@ class NovelTargetHandler {
281
283
  aggregate.skipped += result.skipped;
282
284
  aggregate.alreadyDownloaded += result.alreadyDownloaded;
283
285
  aggregate.filteredOut += result.filteredOut;
286
+ if (result.downloaded > 0) {
287
+ this.markInventorySubmittedFromOutcomes(target, (0, inventory_1.inventoryTopic)(target), 'novel');
288
+ }
284
289
  if (aggregate.scan.outages.length > 0) {
285
290
  // A dead token / dead database / dead network is not "no matching
286
291
  // novel": stop looking back and let the job fail/retry.
287
292
  break;
288
293
  }
289
294
  }
295
+ const topic = (0, inventory_1.inventoryTopic)(target);
296
+ await this.tryInventoryFallback(target, topic, aggregate);
297
+ if (this.supplyRep) {
298
+ this.supplyRep = (0, inventory_1.attachInventoryReport)(this.database, target, topic, this.supplyRep);
299
+ }
290
300
  this.scan = aggregate.scan;
291
301
  await this.handleDownloadResult(aggregate, target, 'topic', totalFound, checkedDays);
292
302
  }
303
+ /**
304
+ * Phase 5 fallback: claim pending novel candidates and publish them into the
305
+ * existing aggregate day-scan (same idempotent pipeline as a normal day).
306
+ */
307
+ async tryInventoryFallback(target, topic, aggregate) {
308
+ const policy = (0, inventory_1.inventoryPolicy)(target);
309
+ if (!policy.enabled || policy.fallback === false)
310
+ return;
311
+ // Only a fully-empty fresh/lookback window may draw from the reserve; a
312
+ // partial day stays on fresh supply rather than padding beyond the limit.
313
+ if (aggregate.downloaded > 0)
314
+ return;
315
+ const targetId = (0, inventory_1.inventoryTargetId)(target);
316
+ const repo = this.database.candidateInventory;
317
+ while (aggregate.downloaded < (target.limit || 1)) {
318
+ const row = repo.claimNext({ topic, targetId, reserveSize: policy.reserveSize, date: (0, pixiv_date_utils_1.getTodayDate)() });
319
+ if (!row)
320
+ break;
321
+ const id = Number(row.pixivId);
322
+ if (!Number.isFinite(id)) {
323
+ repo.markFiltered(row.pixivId, row.workType, topic, targetId);
324
+ continue;
325
+ }
326
+ let detail;
327
+ try {
328
+ detail = await this.client.getNovelDetail(id);
329
+ }
330
+ catch (error) {
331
+ this.logError(error, `Inventory candidate novel ${id} could not be fetched`);
332
+ repo.markSelectedBackToPending(row.pixivId, row.workType, topic, targetId);
333
+ continue;
334
+ }
335
+ const attemptTarget = {
336
+ ...target,
337
+ limit: (target.limit || 1) - aggregate.downloaded,
338
+ };
339
+ const result = await this.pipeline.run([detail], attemptTarget, 'novel', (novel, tag) => this.downloadAndDeliver(novel, tag, attemptTarget));
340
+ aggregate.scan = (0, TargetOutcome_1.mergeScanSummaries)(aggregate.scan, result.scan);
341
+ aggregate.downloaded += result.downloaded;
342
+ aggregate.skipped += result.skipped;
343
+ aggregate.alreadyDownloaded += result.alreadyDownloaded;
344
+ aggregate.filteredOut += result.filteredOut;
345
+ if (result.downloaded > 0) {
346
+ (0, inventory_1.markInventoryAttempt)(this.database, target, topic, row.pixivId, 'novel', 'submitted');
347
+ continue;
348
+ }
349
+ if (aggregate.scan.outages.length > 0)
350
+ return;
351
+ (0, inventory_1.markInventoryAttempt)(this.database, target, topic, row.pixivId, 'novel', 'filtered');
352
+ }
353
+ }
354
+ /** Mark inventory rows whose work reached a delivery/stored outcome. */
355
+ markInventorySubmittedFromOutcomes(target, topic, workType) {
356
+ if (!(0, inventory_1.inventoryPolicy)(target).enabled)
357
+ return;
358
+ for (const outcome of this.outcomes) {
359
+ if (outcome.kind === 'delivery_pending' || outcome.kind === 'submitted' || outcome.kind === 'stored') {
360
+ (0, inventory_1.markInventoryAttempt)(this.database, target, topic, outcome.workId, workType, 'submitted');
361
+ }
362
+ }
363
+ }
293
364
  resolveTopicDay(target) {
294
365
  return target.date === 'TODAY'
295
366
  ? (0, pixiv_date_utils_1.getTodayDate)()
@@ -0,0 +1,29 @@
1
+ import type { TargetConfig } from '../config';
2
+ import type { IDatabase } from '../interfaces/IDatabase';
3
+ import type { CandidateSupplyReport } from '../scheduler/TargetOutcome';
4
+ /** Defaults and bounds for the optional Phase 5 inventory. */
5
+ export interface InventoryPolicy {
6
+ enabled: boolean;
7
+ maxAgeDays: number;
8
+ reserveSize: number;
9
+ fallback: boolean;
10
+ }
11
+ export declare function inventoryPolicy(target: TargetConfig): InventoryPolicy;
12
+ export declare function inventoryTopic(target: TargetConfig): string;
13
+ export declare function inventoryTargetId(target: TargetConfig): string;
14
+ /**
15
+ * Phase 5: after a topic scan yields eligible works that were NOT delivered by
16
+ * this run's lookback loop, upsert them as idle pending inventory. Works that
17
+ * are already recorded as downloaded/delivered are skipped by the caller
18
+ * upstream (selectWorks already removed download-history rows); this method is
19
+ * deliberately idempotent.
20
+ */
21
+ export declare function recordInventoryCandidates(database: IDatabase, target: TargetConfig, topic: string, date: string, works: Array<{
22
+ id: number;
23
+ type?: string;
24
+ }>, workType: 'illustration' | 'novel'): void;
25
+ /** Enrich a candidate report with the durable pending-count snapshot. */
26
+ export declare function attachInventoryReport(database: IDatabase, target: TargetConfig, topic: string, report: CandidateSupplyReport): CandidateSupplyReport;
27
+ /** Marks a claimed inventory row after a delivery attempt. */
28
+ export declare function markInventoryAttempt(database: IDatabase, target: TargetConfig, topic: string, workId: string, workType: 'illustration' | 'novel', outcome: 'submitted' | 'filtered' | 'pending'): void;
29
+ //# sourceMappingURL=inventory.d.ts.map
@@ -0,0 +1,80 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.inventoryPolicy = inventoryPolicy;
4
+ exports.inventoryTopic = inventoryTopic;
5
+ exports.inventoryTargetId = inventoryTargetId;
6
+ exports.recordInventoryCandidates = recordInventoryCandidates;
7
+ exports.attachInventoryReport = attachInventoryReport;
8
+ exports.markInventoryAttempt = markInventoryAttempt;
9
+ function inventoryPolicy(target) {
10
+ const cfg = target.topicProfile?.inventory;
11
+ return {
12
+ enabled: cfg?.enabled === true,
13
+ maxAgeDays: cfg?.maxAgeDays ?? 30,
14
+ reserveSize: Math.min(Math.max(cfg?.reserveSize ?? 20, 1), 100),
15
+ fallback: cfg?.fallback ?? true,
16
+ };
17
+ }
18
+ function inventoryTopic(target) {
19
+ return (target.topicProfile?.primary?.[0] ?? target.topic ?? '').trim();
20
+ }
21
+ function inventoryTargetId(target) {
22
+ return target.id ?? inventoryTopic(target);
23
+ }
24
+ /**
25
+ * Phase 5: after a topic scan yields eligible works that were NOT delivered by
26
+ * this run's lookback loop, upsert them as idle pending inventory. Works that
27
+ * are already recorded as downloaded/delivered are skipped by the caller
28
+ * upstream (selectWorks already removed download-history rows); this method is
29
+ * deliberately idempotent.
30
+ */
31
+ function recordInventoryCandidates(database, target, topic, date, works, workType) {
32
+ const policy = inventoryPolicy(target);
33
+ if (!policy.enabled)
34
+ return;
35
+ const targetId = inventoryTargetId(target);
36
+ for (const work of works) {
37
+ const id = String(work?.id);
38
+ if (!id)
39
+ continue;
40
+ database.candidateInventory.upsert({
41
+ pixivId: id,
42
+ workType,
43
+ topic,
44
+ targetId,
45
+ snapshot: work,
46
+ date,
47
+ maxAgeDays: policy.maxAgeDays,
48
+ });
49
+ }
50
+ }
51
+ /** Enrich a candidate report with the durable pending-count snapshot. */
52
+ function attachInventoryReport(database, target, topic, report) {
53
+ const policy = inventoryPolicy(target);
54
+ if (!policy.enabled)
55
+ return report;
56
+ const summary = database.candidateInventory.pendingSummary(topic, inventoryTargetId(target));
57
+ return {
58
+ ...report,
59
+ inventory: {
60
+ pendingCount: summary.count,
61
+ reserveSize: policy.reserveSize,
62
+ maxAgeDays: policy.maxAgeDays,
63
+ oldestSeenDate: summary.oldestSeenDate,
64
+ },
65
+ };
66
+ }
67
+ /** Marks a claimed inventory row after a delivery attempt. */
68
+ function markInventoryAttempt(database, target, topic, workId, workType, outcome) {
69
+ if (!inventoryPolicy(target).enabled)
70
+ return;
71
+ const targetId = inventoryTargetId(target);
72
+ const repo = database.candidateInventory;
73
+ if (outcome === 'submitted')
74
+ repo.markSubmitted(workId, workType, topic, targetId);
75
+ else if (outcome === 'filtered')
76
+ repo.markFiltered(workId, workType, topic, targetId);
77
+ else
78
+ repo.markSelectedBackToPending(workId, workType, topic, targetId);
79
+ }
80
+ //# sourceMappingURL=inventory.js.map
@@ -1,4 +1,5 @@
1
1
  import { AccessTokenStore, DownloadRecordInput, ExecutionStatus, SchedulerExecutionRecord } from '../storage/Database';
2
+ import { CandidateInventoryRepository } from '../storage/repositories/CandidateInventoryRepository';
2
3
  /**
3
4
  * Interface for database operations
4
5
  * Provides abstraction for data persistence
@@ -123,5 +124,7 @@ export interface IDatabase {
123
124
  };
124
125
  countSince(botId: string | null, hours: number): number;
125
126
  };
127
+ /** Phase 5 durable CandidateInventory (待发池). */
128
+ readonly candidateInventory: CandidateInventoryRepository;
126
129
  }
127
130
  //# sourceMappingURL=IDatabase.d.ts.map
package/dist/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "type": "commonjs",
3
3
  "name": "pixivflow",
4
- "version": "2.29.0",
4
+ "version": "2.30.0",
5
5
  "private": true
6
6
  }
@@ -109,6 +109,17 @@ export interface CandidateSupplyReason {
109
109
  code: string;
110
110
  count: number;
111
111
  }
112
+ /** Phase 5: durable 待发池 snapshot for sparse topics (only when enabled). */
113
+ export interface CandidateInventoryReport {
114
+ /** Rows still usable for fallback at report time. */
115
+ pendingCount: number;
116
+ /** Target's configured max retained rows. */
117
+ reserveSize: number;
118
+ /** Max age in days before a pending candidate expires. */
119
+ maxAgeDays: number;
120
+ /** Earliest still-pending first-seen date (YYYY-MM-DD), null when empty. */
121
+ oldestSeenDate: string | null;
122
+ }
112
123
  /** Candidate-supply observability snapshot (Phase 1 Candidate Report). */
113
124
  export interface CandidateSupplyReport {
114
125
  /** Total works surfaced by the topic search before any filtering. */
@@ -119,6 +130,8 @@ export interface CandidateSupplyReport {
119
130
  rejected: number;
120
131
  /** Why candidates were rejected, by reason code (extensible). */
121
132
  reasons: CandidateSupplyReason[];
133
+ /** Phase 5: durable 待发池 reserve (optional, present only when enabled). */
134
+ inventory?: CandidateInventoryReport;
122
135
  }
123
136
  /**
124
137
  * A freshly-harvested upstream funnel with no candidates selected yet.
@@ -20,7 +20,7 @@ exports.terminalReasonFor = terminalReasonFor;
20
20
  * A freshly-harvested upstream funnel with no candidates selected yet.
21
21
  */
22
22
  function emptyCandidateSupplyReport() {
23
- return { fetched: 0, selected: 0, rejected: 0, reasons: [] };
23
+ return { fetched: 0, selected: 0, rejected: 0, reasons: [], inventory: undefined };
24
24
  }
25
25
  /**
26
26
  * Fold two supply snapshots of the SAME logical target together, so a
@@ -43,6 +43,7 @@ function mergeCandidateSupplyReports(first, second) {
43
43
  selected: first.selected + second.selected,
44
44
  rejected: Math.max(0, first.fetched + second.fetched - (first.selected + second.selected)),
45
45
  reasons,
46
+ inventory: second.inventory ?? first.inventory,
46
47
  };
47
48
  }
48
49
  /**
@@ -68,6 +69,7 @@ function withScanSkips(report, scan) {
68
69
  selected,
69
70
  rejected: Math.max(0, base.fetched - selected),
70
71
  reasons,
72
+ inventory: base.inventory,
71
73
  };
72
74
  }
73
75
  /**
@@ -5,6 +5,7 @@ import { OutboxRepository } from './repositories/OutboxRepository';
5
5
  import { MetadataRepository } from './repositories/MetadataRepository';
6
6
  import { SQLiteRateLimitStateStore } from './repositories/RateLimitStateRepository';
7
7
  import { SystemErrorRepository } from './repositories/SystemErrorRepository';
8
+ import { CandidateInventoryRepository } from './repositories/CandidateInventoryRepository';
8
9
  export interface AccessTokenStore {
9
10
  accessToken: string;
10
11
  expiresAt: number;
@@ -48,6 +49,7 @@ export declare class Database implements IDatabase {
48
49
  private metadataRepo;
49
50
  private rateLimitStateStore;
50
51
  private systemErrorRepo;
52
+ private candidateInventoryRepo;
51
53
  constructor(databasePath: string);
52
54
  migrate(): void;
53
55
  /** Absolute path of the SQLite file (used to locate sibling cache dirs). */
@@ -64,6 +66,8 @@ export declare class Database implements IDatabase {
64
66
  get rateLimitState(): SQLiteRateLimitStateStore;
65
67
  /** Durable system-error ledger (observability; never throws). */
66
68
  get systemErrors(): SystemErrorRepository;
69
+ /** Phase 5 CandidateInventory (待发池) for sparse topics. */
70
+ get candidateInventory(): CandidateInventoryRepository;
67
71
  /** Raw transactional boundary for atomic multi-table intents. */
68
72
  transaction<T>(fn: () => T): T;
69
73
  /** Expose a prepared-statement helper if needed by services (pragmas etc). */
@@ -18,6 +18,7 @@ const OutboxRepository_1 = require("./repositories/OutboxRepository");
18
18
  const MetadataRepository_1 = require("./repositories/MetadataRepository");
19
19
  const RateLimitStateRepository_1 = require("./repositories/RateLimitStateRepository");
20
20
  const SystemErrorRepository_1 = require("./repositories/SystemErrorRepository");
21
+ const CandidateInventoryRepository_1 = require("./repositories/CandidateInventoryRepository");
21
22
  const NodeSqliteDriver_1 = require("./drivers/NodeSqliteDriver");
22
23
  class Database {
23
24
  databasePath;
@@ -35,6 +36,7 @@ class Database {
35
36
  metadataRepo;
36
37
  rateLimitStateStore;
37
38
  systemErrorRepo;
39
+ candidateInventoryRepo;
38
40
  constructor(databasePath) {
39
41
  this.databasePath = databasePath;
40
42
  try {
@@ -63,6 +65,7 @@ class Database {
63
65
  this.metadataRepo = new MetadataRepository_1.MetadataRepository(this.db);
64
66
  this.rateLimitStateStore = new RateLimitStateRepository_1.SQLiteRateLimitStateStore(this.db);
65
67
  this.systemErrorRepo = new SystemErrorRepository_1.SystemErrorRepository(this.db);
68
+ this.candidateInventoryRepo = new CandidateInventoryRepository_1.CandidateInventoryRepository(this.db);
66
69
  }
67
70
  catch (error) {
68
71
  throw new errors_1.DatabaseError(`Failed to initialize database at ${this.databasePath}`, error instanceof Error ? error : undefined);
@@ -102,6 +105,10 @@ class Database {
102
105
  get systemErrors() {
103
106
  return this.systemErrorRepo;
104
107
  }
108
+ /** Phase 5 CandidateInventory (待发池) for sparse topics. */
109
+ get candidateInventory() {
110
+ return this.candidateInventoryRepo;
111
+ }
105
112
  /** Raw transactional boundary for atomic multi-table intents. */
106
113
  transaction(fn) {
107
114
  return this.db.transaction(fn)();
@@ -222,6 +222,25 @@ class DatabaseMigration {
222
222
  scope TEXT PRIMARY KEY,
223
223
  state TEXT NOT NULL,
224
224
  updated_at INTEGER NOT NULL
225
+ )`,
226
+ // Phase 5 CandidateInventory: durable 待发池 for sparse topics. Same
227
+ // database and transaction world as the Slot Ledger; never a second
228
+ // state authority. Only populated when target.topicProfile.inventory.enabled.
229
+ `CREATE TABLE IF NOT EXISTS candidate_inventory (
230
+ pixiv_id TEXT NOT NULL,
231
+ work_type TEXT NOT NULL,
232
+ topic TEXT NOT NULL,
233
+ target_id TEXT NOT NULL,
234
+ status TEXT NOT NULL DEFAULT 'pending',
235
+ snapshot_json TEXT NOT NULL,
236
+ first_seen_date TEXT NOT NULL,
237
+ last_seen_date TEXT NOT NULL,
238
+ seen_count INTEGER NOT NULL DEFAULT 1,
239
+ attempt_count INTEGER NOT NULL DEFAULT 0,
240
+ expires_at TEXT NOT NULL,
241
+ created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
242
+ updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
243
+ PRIMARY KEY (pixiv_id, work_type, topic, target_id)
225
244
  )`,
226
245
  // Durable error event ledger for observability (structured error
227
246
  // taxonomy; populated by download/system handlers and shown in the
@@ -244,6 +263,7 @@ class DatabaseMigration {
244
263
  resolved_at DATETIME
245
264
  )`,
246
265
  `CREATE INDEX IF NOT EXISTS idx_system_errors_bot_created ON system_errors(bot_id, created_at)`,
266
+ `CREATE INDEX IF NOT EXISTS idx_cinventory_pending ON candidate_inventory(status, first_seen_date, expires_at)`,
247
267
  ];
248
268
  // Phase 1: create tables (idempotent). Must run before any PRAGMA-based
249
269
  // column check, otherwise a fresh DB would report the table as missing and
@@ -0,0 +1,51 @@
1
+ import { BaseRepository } from './BaseRepository';
2
+ export type CandidateInventoryStatus = 'pending' | 'selected' | 'submitted' | 'filtered' | 'expired';
3
+ export interface CandidateInventoryRow {
4
+ pixivId: string;
5
+ workType: 'illustration' | 'novel';
6
+ topic: string;
7
+ targetId: string;
8
+ status: CandidateInventoryStatus;
9
+ snapshotJson: string;
10
+ firstSeenDate: string;
11
+ lastSeenDate: string;
12
+ seenCount: number;
13
+ attemptCount: number;
14
+ expiresAt: string;
15
+ createdAt: string;
16
+ updatedAt: string;
17
+ }
18
+ export interface CandidateInventorySnapshot {
19
+ pixivId: string;
20
+ workType: 'illustration' | 'novel';
21
+ topic: string;
22
+ targetId: string;
23
+ snapshot: unknown;
24
+ date: string;
25
+ maxAgeDays: number;
26
+ }
27
+ /** Phase 5 durable 待发池 — same SQLite database as the Slot Ledger. */
28
+ export declare class CandidateInventoryRepository extends BaseRepository {
29
+ upsert(input: CandidateInventorySnapshot): void;
30
+ /** Marks a claimed candidate and returns its snapshot for the pipeline. */
31
+ claimNext(input: {
32
+ topic: string;
33
+ targetId: string;
34
+ reserveSize: number;
35
+ date: string;
36
+ }): CandidateInventoryRow | null;
37
+ markSubmitted(pixivId: string, workType: string, topic: string, targetId: string): void;
38
+ markFiltered(pixivId: string, workType: string, topic: string, targetId: string): void;
39
+ markSelectedBackToPending(pixivId: string, workType: string, topic: string, targetId: string): void;
40
+ countPending(topic: string, targetId: string): number;
41
+ pendingSummary(topic: string, targetId: string): {
42
+ count: number;
43
+ oldestSeenDate: string | null;
44
+ };
45
+ /** Sweep rows past maxAgeDays; idempotent per scheduled scan. */
46
+ evictExpired(topic: string, targetId: string): number;
47
+ private toRow;
48
+ private todayUtc;
49
+ private addDaysUtc;
50
+ }
51
+ //# sourceMappingURL=CandidateInventoryRepository.d.ts.map
@@ -0,0 +1,129 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.CandidateInventoryRepository = void 0;
4
+ const BaseRepository_1 = require("./BaseRepository");
5
+ /** Phase 5 durable 待发池 — same SQLite database as the Slot Ledger. */
6
+ class CandidateInventoryRepository extends BaseRepository_1.BaseRepository {
7
+ upsert(input) {
8
+ const expiresAt = this.addDaysUtc(input.date, input.maxAgeDays);
9
+ this.db
10
+ .prepare(`INSERT INTO candidate_inventory
11
+ (pixiv_id, work_type, topic, target_id, status, snapshot_json,
12
+ first_seen_date, last_seen_date, seen_count, attempt_count, expires_at)
13
+ VALUES
14
+ (@pixivId, @workType, @topic, @targetId, 'pending', @snapshot,
15
+ @date, @date, 1, 0, @expiresAt)
16
+ ON CONFLICT(pixiv_id, work_type, topic, target_id) DO UPDATE SET
17
+ snapshot_json = excluded.snapshot_json,
18
+ last_seen_date = excluded.last_seen_date,
19
+ seen_count = candidate_inventory.seen_count + 1,
20
+ status = CASE
21
+ WHEN candidate_inventory.status IN ('selected','submitted','filtered','expired')
22
+ THEN candidate_inventory.status
23
+ ELSE 'pending'
24
+ END,
25
+ expires_at = excluded.expires_at,
26
+ updated_at = CURRENT_TIMESTAMP`)
27
+ .run({
28
+ pixivId: input.pixivId,
29
+ workType: input.workType,
30
+ topic: input.topic,
31
+ targetId: input.targetId,
32
+ snapshot: JSON.stringify(input.snapshot),
33
+ date: input.date,
34
+ expiresAt,
35
+ });
36
+ }
37
+ /** Marks a claimed candidate and returns its snapshot for the pipeline. */
38
+ claimNext(input) {
39
+ const rows = this.db
40
+ .prepare(`SELECT * FROM candidate_inventory
41
+ WHERE topic = ? AND target_id = ?
42
+ AND status = 'pending'
43
+ AND expires_at >= ?
44
+ ORDER BY first_seen_date ASC, seen_count ASC
45
+ LIMIT ?`)
46
+ .all(input.topic, input.targetId, this.addDaysUtc(input.date, 0), input.reserveSize);
47
+ for (const row of rows) {
48
+ const updated = this.db
49
+ .prepare(`UPDATE candidate_inventory
50
+ SET status = 'selected', attempt_count = attempt_count + 1, updated_at = CURRENT_TIMESTAMP
51
+ WHERE pixiv_id = ? AND work_type = ? AND topic = ? AND target_id = ? AND status = 'pending'`)
52
+ .run(row.pixiv_id, row.work_type, row.topic, row.target_id);
53
+ if (updated.changes > 0)
54
+ return this.toRow(row);
55
+ }
56
+ return null;
57
+ }
58
+ markSubmitted(pixivId, workType, topic, targetId) {
59
+ this.db
60
+ .prepare(`UPDATE candidate_inventory
61
+ SET status = 'submitted', updated_at = CURRENT_TIMESTAMP
62
+ WHERE pixiv_id = ? AND work_type = ? AND topic = ? AND target_id = ?`)
63
+ .run(pixivId, workType, topic, targetId);
64
+ }
65
+ markFiltered(pixivId, workType, topic, targetId) {
66
+ this.db
67
+ .prepare(`UPDATE candidate_inventory
68
+ SET status = 'filtered', updated_at = CURRENT_TIMESTAMP
69
+ WHERE pixiv_id = ? AND work_type = ? AND topic = ? AND target_id = ?`)
70
+ .run(pixivId, workType, topic, targetId);
71
+ }
72
+ markSelectedBackToPending(pixivId, workType, topic, targetId) {
73
+ this.db
74
+ .prepare(`UPDATE candidate_inventory
75
+ SET status = 'pending', updated_at = CURRENT_TIMESTAMP
76
+ WHERE pixiv_id = ? AND work_type = ? AND topic = ? AND target_id = ? AND status = 'selected'`)
77
+ .run(pixivId, workType, topic, targetId);
78
+ }
79
+ countPending(topic, targetId) {
80
+ const rows = this.db
81
+ .prepare(`SELECT COUNT(*) AS n FROM candidate_inventory WHERE topic = ? AND target_id = ? AND status = 'pending' AND expires_at >= ?`)
82
+ .get(topic, targetId, this.todayUtc());
83
+ return Number(rows?.n ?? 0);
84
+ }
85
+ pendingSummary(topic, targetId) {
86
+ const row = this.db
87
+ .prepare(`SELECT COUNT(*) AS n, MIN(first_seen_date) AS oldest
88
+ FROM candidate_inventory
89
+ WHERE topic = ? AND target_id = ? AND status = 'pending' AND expires_at >= ?`)
90
+ .get(topic, targetId, this.todayUtc());
91
+ return { count: Number(row?.n ?? 0), oldestSeenDate: row?.oldest ?? null };
92
+ }
93
+ /** Sweep rows past maxAgeDays; idempotent per scheduled scan. */
94
+ evictExpired(topic, targetId) {
95
+ const info = this.db
96
+ .prepare(`UPDATE candidate_inventory
97
+ SET status = 'expired', updated_at = CURRENT_TIMESTAMP
98
+ WHERE topic = ? AND target_id = ? AND status IN ('pending','selected') AND expires_at < ?`)
99
+ .run(topic, targetId, this.todayUtc());
100
+ return info.changes;
101
+ }
102
+ toRow(row) {
103
+ return {
104
+ pixivId: String(row.pixiv_id),
105
+ workType: row.work_type,
106
+ topic: row.topic,
107
+ targetId: row.target_id,
108
+ status: row.status,
109
+ snapshotJson: row.snapshot_json,
110
+ firstSeenDate: row.first_seen_date,
111
+ lastSeenDate: row.last_seen_date,
112
+ seenCount: Number(row.seen_count ?? 1),
113
+ attemptCount: Number(row.attempt_count ?? 0),
114
+ expiresAt: row.expires_at,
115
+ createdAt: row.created_at,
116
+ updatedAt: row.updated_at,
117
+ };
118
+ }
119
+ todayUtc() {
120
+ return new Date().toISOString().slice(0, 10);
121
+ }
122
+ addDaysUtc(date, days) {
123
+ const d = new Date(`${date}T00:00:00.000Z`);
124
+ d.setUTCDate(d.getUTCDate() + days);
125
+ return d.toISOString().slice(0, 10);
126
+ }
127
+ }
128
+ exports.CandidateInventoryRepository = CandidateInventoryRepository;
129
+ //# sourceMappingURL=CandidateInventoryRepository.js.map
package/dist/version.js CHANGED
@@ -2,5 +2,5 @@
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.BUILD = void 0;
4
4
  // GENERATED by scripts/write-version.js — do not edit manually.
5
- exports.BUILD = { version: '2.29.0', commit: '13e234a69cd2' };
5
+ exports.BUILD = { version: '2.30.0', commit: '153116bd9b08' };
6
6
  //# sourceMappingURL=version.js.map
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "type": "commonjs",
3
3
  "name": "pixivflow-webui-backend",
4
- "version": "2.29.0",
4
+ "version": "2.30.0",
5
5
  "description": "PixivFlow WebUI Backend - CommonJS module"
6
6
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "pixivflow",
3
- "version": "2.29.0",
3
+ "version": "2.30.0",
4
4
  "description": "🎨 Pixiv 下载、筛选与自动收集工具 - 批量下载插画和小说、按标签/热度/日期筛选、定时任务与可靠 HTTP 交付 | Pixiv downloader and automation toolkit with filtering, scheduling and reliable HTTP delivery",
5
5
  "repository": {
6
6
  "type": "git",