pixivflow 2.29.0 → 2.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config/types.d.ts +31 -0
- package/dist/config/validation.js +35 -0
- package/dist/download/handlers/IllustrationTargetHandler.d.ts +8 -0
- package/dist/download/handlers/IllustrationTargetHandler.js +74 -0
- package/dist/download/handlers/NovelTargetHandler.d.ts +7 -0
- package/dist/download/handlers/NovelTargetHandler.js +71 -0
- package/dist/download/inventory.d.ts +29 -0
- package/dist/download/inventory.js +80 -0
- package/dist/interfaces/IDatabase.d.ts +3 -0
- package/dist/package.json +1 -1
- package/dist/scheduler/TargetOutcome.d.ts +13 -0
- package/dist/scheduler/TargetOutcome.js +3 -1
- package/dist/storage/Database.d.ts +4 -0
- package/dist/storage/Database.js +7 -0
- package/dist/storage/DatabaseMigration.js +20 -0
- package/dist/storage/repositories/CandidateInventoryRepository.d.ts +51 -0
- package/dist/storage/repositories/CandidateInventoryRepository.js +129 -0
- package/dist/version.js +1 -1
- package/dist/webui/package.json +1 -1
- package/package.json +1 -1
package/dist/config/types.d.ts
CHANGED
|
@@ -27,6 +27,35 @@ export interface CandidateCollectionConfig {
|
|
|
27
27
|
/** Minimum metadata-topic score to survive filtering (default 0.35). */
|
|
28
28
|
minMetadataScore?: number;
|
|
29
29
|
}
|
|
30
|
+
/** Phase 4: topic-supply strategy (per-target, optional, default off). */
|
|
31
|
+
export interface TopicProfileStrategyConfig {
|
|
32
|
+
/** Relative weight for publishing fresher works first (0..1, default 0.4). */
|
|
33
|
+
freshnessWeight?: number;
|
|
34
|
+
/** Relative weight for publishing higher-popularity works first (0..1, default 0.3). */
|
|
35
|
+
popularityWeight?: number;
|
|
36
|
+
}
|
|
37
|
+
/** Phase 5: durable candidate inventory ("待发池") for a target. */
|
|
38
|
+
export interface CandidateInventoryConfig {
|
|
39
|
+
/** Master switch. Default false: no collection, no fallback, no new columns read. */
|
|
40
|
+
enabled?: boolean;
|
|
41
|
+
/** Max age (days) a pending candidate may stay usable for fallback. Default 30. */
|
|
42
|
+
maxAgeDays?: number;
|
|
43
|
+
/** Max pending rows retained per (topic,target). Default 20. */
|
|
44
|
+
reserveSize?: number;
|
|
45
|
+
/** When the fresh/lookback scan is empty, publish oldest pending inventory. Default true. */
|
|
46
|
+
fallback?: boolean;
|
|
47
|
+
}
|
|
48
|
+
/** Phase 4: TopicProfile (seed + related tags + supply strategy). */
|
|
49
|
+
export interface TopicProfileConfig {
|
|
50
|
+
/** Primary seed tag(s); first item is the canonical topic when topic is unset. */
|
|
51
|
+
primary?: string[];
|
|
52
|
+
/** Related tags that strengthen topic membership evidence. */
|
|
53
|
+
related?: string[];
|
|
54
|
+
/** Ranking strategy knobs (Phase 4; wired after CandidateInventory). */
|
|
55
|
+
strategy?: TopicProfileStrategyConfig;
|
|
56
|
+
/** CandidateInventory policy (Phase 5). */
|
|
57
|
+
inventory?: CandidateInventoryConfig;
|
|
58
|
+
}
|
|
30
59
|
/** Behaviour when a target cannot produce the requested number of works. */
|
|
31
60
|
export interface NoMatchPolicyConfig {
|
|
32
61
|
/**
|
|
@@ -178,6 +207,8 @@ export interface TargetConfig {
|
|
|
178
207
|
candidateCollection?: CandidateCollectionConfig;
|
|
179
208
|
/** Bounded fallback and notification policy for an empty result. */
|
|
180
209
|
noMatchPolicy?: NoMatchPolicyConfig;
|
|
210
|
+
/** Phase 4/5: topic supply profile + candidate inventory policy (all optional). */
|
|
211
|
+
topicProfile?: TopicProfileConfig;
|
|
181
212
|
/**
|
|
182
213
|
* Ranking mode (only used when mode='ranking')
|
|
183
214
|
* - 'day': Daily ranking
|
|
@@ -240,6 +240,41 @@ function validateConfig(config, location, databasePath) {
|
|
|
240
240
|
if (target.noMatchPolicy?.notify !== undefined && typeof target.noMatchPolicy.notify !== 'boolean') {
|
|
241
241
|
errors.push(`targets[${index}].noMatchPolicy.notify: Must be a boolean`);
|
|
242
242
|
}
|
|
243
|
+
if (target.topicProfile) {
|
|
244
|
+
const tp = target.topicProfile;
|
|
245
|
+
if (tp.primary !== undefined && (!Array.isArray(tp.primary) || tp.primary.length === 0 || tp.primary.some((x) => typeof x !== 'string' || !x.trim()))) {
|
|
246
|
+
errors.push(`targets[${index}].topicProfile.primary: Must be a non-empty string array`);
|
|
247
|
+
}
|
|
248
|
+
if (tp.related !== undefined && (!Array.isArray(tp.related) || tp.related.some((x) => typeof x !== 'string' || !x.trim()))) {
|
|
249
|
+
errors.push(`targets[${index}].topicProfile.related: Must be a string array`);
|
|
250
|
+
}
|
|
251
|
+
const st = tp.strategy;
|
|
252
|
+
if (st) {
|
|
253
|
+
for (const field of ['freshnessWeight', 'popularityWeight']) {
|
|
254
|
+
const value = st[field];
|
|
255
|
+
if (value !== undefined && (typeof value !== 'number' || value < 0 || value > 1)) {
|
|
256
|
+
errors.push(`targets[${index}].topicProfile.strategy.${field}: Must be a number between 0 and 1`);
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
const inv = tp.inventory;
|
|
261
|
+
if (inv) {
|
|
262
|
+
if (inv.enabled !== undefined && typeof inv.enabled !== 'boolean') {
|
|
263
|
+
errors.push(`targets[${index}].topicProfile.inventory.enabled: Must be a boolean`);
|
|
264
|
+
}
|
|
265
|
+
const bound = (field) => {
|
|
266
|
+
const value = inv[field];
|
|
267
|
+
if (value !== undefined && (typeof value !== 'number' || !Number.isInteger(value) || value < 1 || value > 365)) {
|
|
268
|
+
errors.push(`targets[${index}].topicProfile.inventory.${field}: Must be an integer between 1 and 365`);
|
|
269
|
+
}
|
|
270
|
+
};
|
|
271
|
+
bound('maxAgeDays');
|
|
272
|
+
bound('reserveSize');
|
|
273
|
+
if (inv.fallback !== undefined && typeof inv.fallback !== 'boolean') {
|
|
274
|
+
errors.push(`targets[${index}].topicProfile.inventory.fallback: Must be a boolean`);
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
}
|
|
243
278
|
if (target.noMatchPolicy?.notify === true) {
|
|
244
279
|
const deliveryTarget = target.delivery?.target?.trim();
|
|
245
280
|
const notifyTarget = deliveryTarget ? config.delivery?.targets?.[deliveryTarget] : undefined;
|
|
@@ -68,6 +68,14 @@ export declare class IllustrationTargetHandler {
|
|
|
68
68
|
private fetchIllustrations;
|
|
69
69
|
private fetchTopicIllustrations;
|
|
70
70
|
private handleTopicWithLookback;
|
|
71
|
+
/**
|
|
72
|
+
* Phase 5 fallback: claim oldest pending candidate(s) from the durable
|
|
73
|
+
* 待发池 and try to publish them exactly like a normal day's pipeline run.
|
|
74
|
+
* Returns true when at least one work reached a delivery/stored outcome.
|
|
75
|
+
*/
|
|
76
|
+
private tryInventoryFallback;
|
|
77
|
+
/** Mark inventory rows whose work reached a delivery/stored outcome. */
|
|
78
|
+
private markInventorySubmittedFromOutcomes;
|
|
71
79
|
private resolveTopicDay;
|
|
72
80
|
private shiftDay;
|
|
73
81
|
private fetchRankingIllustrations;
|
|
@@ -12,6 +12,7 @@ const observability_1 = require("../../observability");
|
|
|
12
12
|
const context_1 = require("../../observability/context");
|
|
13
13
|
const DownloadPlanner_1 = require("../plan/DownloadPlanner");
|
|
14
14
|
const deliveryContext_1 = require("./deliveryContext");
|
|
15
|
+
const inventory_1 = require("../inventory");
|
|
15
16
|
class IllustrationTargetHandler {
|
|
16
17
|
client;
|
|
17
18
|
database;
|
|
@@ -233,6 +234,9 @@ class IllustrationTargetHandler {
|
|
|
233
234
|
const pipeline = this.topicPipelineFactory();
|
|
234
235
|
const { works, selection } = await pipeline.selectWorks(target, 'illustration', day, selectionLimit, target.topicDiscovery ?? {}, target.candidateCollection ?? {});
|
|
235
236
|
logger_1.logger.info(`Topic "${topic}" illustration: tags=${selection.resolvedTagCount} raw=${selection.rawCount} deduped=${selection.dedupedCount} aiExcluded=${selection.aiExcludedCount} accepted=${selection.acceptedCount} candidates=${works.length} target=${limit}`);
|
|
237
|
+
// Phase 5: harvest the accepted-but-not-yet-delivered works into the
|
|
238
|
+
// durable 待发池 (idempotent; only when the target opts in).
|
|
239
|
+
(0, inventory_1.recordInventoryCandidates)(this.database, target, topic, day, works, 'illustration');
|
|
236
240
|
// Candidate Supply Observability: fold this lookback day's upstream funnel
|
|
237
241
|
// into the target's accumulated Candidate Report.
|
|
238
242
|
const report = (0, TargetOutcome_1.emptyCandidateSupplyReport)();
|
|
@@ -270,6 +274,7 @@ class IllustrationTargetHandler {
|
|
|
270
274
|
this.scan = (0, TargetOutcome_1.mergeScanSummaries)(this.scan, result.scan);
|
|
271
275
|
if (result.downloaded > 0) {
|
|
272
276
|
this.handleDownloadResult(result, target, 'topic', illusts.length);
|
|
277
|
+
this.markInventorySubmittedFromOutcomes(target, (0, inventory_1.inventoryTopic)(target), 'illustration');
|
|
273
278
|
return;
|
|
274
279
|
}
|
|
275
280
|
if (this.scan && this.scan.outages.length > 0) {
|
|
@@ -278,6 +283,16 @@ class IllustrationTargetHandler {
|
|
|
278
283
|
return;
|
|
279
284
|
}
|
|
280
285
|
}
|
|
286
|
+
// Phase 5: when the fresh + lookback scan produced nothing, and the target
|
|
287
|
+
// opted into CandidateInventory, publish oldest pending reserve candidates.
|
|
288
|
+
const topic = (0, inventory_1.inventoryTopic)(target);
|
|
289
|
+
if (await this.tryInventoryFallback(target, topic, 'illustration')) {
|
|
290
|
+
return;
|
|
291
|
+
}
|
|
292
|
+
// Report how much reserve remains so the empty result is not a dead end.
|
|
293
|
+
if (this.supplyRep) {
|
|
294
|
+
this.supplyRep = (0, inventory_1.attachInventoryReport)(this.database, target, topic, this.supplyRep);
|
|
295
|
+
}
|
|
281
296
|
const scan = this.scan;
|
|
282
297
|
// An explicit no-eligible-candidate verdict needs something to have been
|
|
283
298
|
// considered. When the scan is empty (nothing surfaced at all) the richer
|
|
@@ -297,6 +312,65 @@ class IllustrationTargetHandler {
|
|
|
297
312
|
...(scan ? { scan } : {}),
|
|
298
313
|
});
|
|
299
314
|
}
|
|
315
|
+
/**
|
|
316
|
+
* Phase 5 fallback: claim oldest pending candidate(s) from the durable
|
|
317
|
+
* 待发池 and try to publish them exactly like a normal day's pipeline run.
|
|
318
|
+
* Returns true when at least one work reached a delivery/stored outcome.
|
|
319
|
+
*/
|
|
320
|
+
async tryInventoryFallback(target, topic, workType) {
|
|
321
|
+
const policy = (0, inventory_1.inventoryPolicy)(target);
|
|
322
|
+
if (!policy.enabled || policy.fallback === false)
|
|
323
|
+
return false;
|
|
324
|
+
const targetId = (0, inventory_1.inventoryTargetId)(target);
|
|
325
|
+
const repo = this.database.candidateInventory;
|
|
326
|
+
while (true) {
|
|
327
|
+
const row = repo.claimNext({ topic, targetId, reserveSize: policy.reserveSize, date: (0, pixiv_date_utils_1.getTodayDate)() });
|
|
328
|
+
if (!row)
|
|
329
|
+
break;
|
|
330
|
+
const id = Number(row.pixivId);
|
|
331
|
+
if (!Number.isFinite(id)) {
|
|
332
|
+
repo.markFiltered(row.pixivId, row.workType, topic, targetId);
|
|
333
|
+
continue;
|
|
334
|
+
}
|
|
335
|
+
let detail;
|
|
336
|
+
try {
|
|
337
|
+
detail = await this.client.getIllustration(id);
|
|
338
|
+
}
|
|
339
|
+
catch (error) {
|
|
340
|
+
this.logError(error, `Inventory candidate illustration ${id} could not be fetched`);
|
|
341
|
+
repo.markSelectedBackToPending(row.pixivId, row.workType, topic, targetId);
|
|
342
|
+
continue;
|
|
343
|
+
}
|
|
344
|
+
const attemptTarget = { ...target };
|
|
345
|
+
const result = await this.pipeline.run([detail], attemptTarget, 'illustration', (illust, tag) => this.downloadAndDeliver(illust, tag, attemptTarget));
|
|
346
|
+
this.scan = (0, TargetOutcome_1.mergeScanSummaries)(this.scan, result.scan);
|
|
347
|
+
if (result.downloaded > 0) {
|
|
348
|
+
this.handleDownloadResult(result, attemptTarget, 'topic', 1);
|
|
349
|
+
(0, inventory_1.markInventoryAttempt)(this.database, target, topic, row.pixivId, 'illustration', 'submitted');
|
|
350
|
+
return true;
|
|
351
|
+
}
|
|
352
|
+
if (this.scan && this.scan.outages.length > 0) {
|
|
353
|
+
// A dead token / dead network is not "no candidate": retry the same
|
|
354
|
+
// row on the next run instead of discarding it.
|
|
355
|
+
(0, inventory_1.markInventoryAttempt)(this.database, target, topic, row.pixivId, 'illustration', 'pending');
|
|
356
|
+
return false;
|
|
357
|
+
}
|
|
358
|
+
// The claimed work is no longer usable (deleted/private/duplicate): drop
|
|
359
|
+
// it permanently and try the next pending row.
|
|
360
|
+
(0, inventory_1.markInventoryAttempt)(this.database, target, topic, row.pixivId, 'illustration', 'filtered');
|
|
361
|
+
}
|
|
362
|
+
return false;
|
|
363
|
+
}
|
|
364
|
+
/** Mark inventory rows whose work reached a delivery/stored outcome. */
|
|
365
|
+
markInventorySubmittedFromOutcomes(target, topic, workType) {
|
|
366
|
+
if (!(0, inventory_1.inventoryPolicy)(target).enabled)
|
|
367
|
+
return;
|
|
368
|
+
for (const outcome of this.outcomes) {
|
|
369
|
+
if (outcome.kind === 'delivery_pending' || outcome.kind === 'submitted' || outcome.kind === 'stored') {
|
|
370
|
+
(0, inventory_1.markInventoryAttempt)(this.database, target, topic, outcome.workId, workType, 'submitted');
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
}
|
|
300
374
|
resolveTopicDay(target) {
|
|
301
375
|
return target.date === 'TODAY'
|
|
302
376
|
? (0, pixiv_date_utils_1.getTodayDate)()
|
|
@@ -67,6 +67,13 @@ export declare class NovelTargetHandler {
|
|
|
67
67
|
private fetchNovels;
|
|
68
68
|
private fetchTopicNovels;
|
|
69
69
|
private handleTopicWithLookback;
|
|
70
|
+
/**
|
|
71
|
+
* Phase 5 fallback: claim pending novel candidates and publish them into the
|
|
72
|
+
* existing aggregate day-scan (same idempotent pipeline as a normal day).
|
|
73
|
+
*/
|
|
74
|
+
private tryInventoryFallback;
|
|
75
|
+
/** Mark inventory rows whose work reached a delivery/stored outcome. */
|
|
76
|
+
private markInventorySubmittedFromOutcomes;
|
|
70
77
|
private resolveTopicDay;
|
|
71
78
|
private shiftDay;
|
|
72
79
|
private fetchRankingNovels;
|
|
@@ -12,6 +12,7 @@ const observability_1 = require("../../observability");
|
|
|
12
12
|
const context_1 = require("../../observability/context");
|
|
13
13
|
const DownloadPlanner_1 = require("../plan/DownloadPlanner");
|
|
14
14
|
const deliveryContext_1 = require("./deliveryContext");
|
|
15
|
+
const inventory_1 = require("../inventory");
|
|
15
16
|
const TelePressRichNovel_1 = require("../../delivery/TelePressRichNovel");
|
|
16
17
|
class NovelTargetHandler {
|
|
17
18
|
client;
|
|
@@ -232,6 +233,7 @@ class NovelTargetHandler {
|
|
|
232
233
|
const pipeline = this.topicPipelineFactory();
|
|
233
234
|
const { works, selection } = await pipeline.selectWorks(target, 'novel', day, selectionLimit, target.topicDiscovery ?? {}, target.candidateCollection ?? {});
|
|
234
235
|
logger_1.logger.info(`Topic "${topic}" novel: tags=${selection.resolvedTagCount} raw=${selection.rawCount} deduped=${selection.dedupedCount} accepted=${selection.acceptedCount} candidates=${works.length} target=${limit}`);
|
|
236
|
+
(0, inventory_1.recordInventoryCandidates)(this.database, target, topic, day, works, 'novel');
|
|
235
237
|
const report = (0, TargetOutcome_1.emptyCandidateSupplyReport)();
|
|
236
238
|
report.fetched = selection.rawCount;
|
|
237
239
|
report.selected = selection.acceptedCount;
|
|
@@ -281,15 +283,84 @@ class NovelTargetHandler {
|
|
|
281
283
|
aggregate.skipped += result.skipped;
|
|
282
284
|
aggregate.alreadyDownloaded += result.alreadyDownloaded;
|
|
283
285
|
aggregate.filteredOut += result.filteredOut;
|
|
286
|
+
if (result.downloaded > 0) {
|
|
287
|
+
this.markInventorySubmittedFromOutcomes(target, (0, inventory_1.inventoryTopic)(target), 'novel');
|
|
288
|
+
}
|
|
284
289
|
if (aggregate.scan.outages.length > 0) {
|
|
285
290
|
// A dead token / dead database / dead network is not "no matching
|
|
286
291
|
// novel": stop looking back and let the job fail/retry.
|
|
287
292
|
break;
|
|
288
293
|
}
|
|
289
294
|
}
|
|
295
|
+
const topic = (0, inventory_1.inventoryTopic)(target);
|
|
296
|
+
await this.tryInventoryFallback(target, topic, aggregate);
|
|
297
|
+
if (this.supplyRep) {
|
|
298
|
+
this.supplyRep = (0, inventory_1.attachInventoryReport)(this.database, target, topic, this.supplyRep);
|
|
299
|
+
}
|
|
290
300
|
this.scan = aggregate.scan;
|
|
291
301
|
await this.handleDownloadResult(aggregate, target, 'topic', totalFound, checkedDays);
|
|
292
302
|
}
|
|
303
|
+
/**
|
|
304
|
+
* Phase 5 fallback: claim pending novel candidates and publish them into the
|
|
305
|
+
* existing aggregate day-scan (same idempotent pipeline as a normal day).
|
|
306
|
+
*/
|
|
307
|
+
async tryInventoryFallback(target, topic, aggregate) {
|
|
308
|
+
const policy = (0, inventory_1.inventoryPolicy)(target);
|
|
309
|
+
if (!policy.enabled || policy.fallback === false)
|
|
310
|
+
return;
|
|
311
|
+
// Only a fully-empty fresh/lookback window may draw from the reserve; a
|
|
312
|
+
// partial day stays on fresh supply rather than padding beyond the limit.
|
|
313
|
+
if (aggregate.downloaded > 0)
|
|
314
|
+
return;
|
|
315
|
+
const targetId = (0, inventory_1.inventoryTargetId)(target);
|
|
316
|
+
const repo = this.database.candidateInventory;
|
|
317
|
+
while (aggregate.downloaded < (target.limit || 1)) {
|
|
318
|
+
const row = repo.claimNext({ topic, targetId, reserveSize: policy.reserveSize, date: (0, pixiv_date_utils_1.getTodayDate)() });
|
|
319
|
+
if (!row)
|
|
320
|
+
break;
|
|
321
|
+
const id = Number(row.pixivId);
|
|
322
|
+
if (!Number.isFinite(id)) {
|
|
323
|
+
repo.markFiltered(row.pixivId, row.workType, topic, targetId);
|
|
324
|
+
continue;
|
|
325
|
+
}
|
|
326
|
+
let detail;
|
|
327
|
+
try {
|
|
328
|
+
detail = await this.client.getNovelDetail(id);
|
|
329
|
+
}
|
|
330
|
+
catch (error) {
|
|
331
|
+
this.logError(error, `Inventory candidate novel ${id} could not be fetched`);
|
|
332
|
+
repo.markSelectedBackToPending(row.pixivId, row.workType, topic, targetId);
|
|
333
|
+
continue;
|
|
334
|
+
}
|
|
335
|
+
const attemptTarget = {
|
|
336
|
+
...target,
|
|
337
|
+
limit: (target.limit || 1) - aggregate.downloaded,
|
|
338
|
+
};
|
|
339
|
+
const result = await this.pipeline.run([detail], attemptTarget, 'novel', (novel, tag) => this.downloadAndDeliver(novel, tag, attemptTarget));
|
|
340
|
+
aggregate.scan = (0, TargetOutcome_1.mergeScanSummaries)(aggregate.scan, result.scan);
|
|
341
|
+
aggregate.downloaded += result.downloaded;
|
|
342
|
+
aggregate.skipped += result.skipped;
|
|
343
|
+
aggregate.alreadyDownloaded += result.alreadyDownloaded;
|
|
344
|
+
aggregate.filteredOut += result.filteredOut;
|
|
345
|
+
if (result.downloaded > 0) {
|
|
346
|
+
(0, inventory_1.markInventoryAttempt)(this.database, target, topic, row.pixivId, 'novel', 'submitted');
|
|
347
|
+
continue;
|
|
348
|
+
}
|
|
349
|
+
if (aggregate.scan.outages.length > 0)
|
|
350
|
+
return;
|
|
351
|
+
(0, inventory_1.markInventoryAttempt)(this.database, target, topic, row.pixivId, 'novel', 'filtered');
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
/** Mark inventory rows whose work reached a delivery/stored outcome. */
|
|
355
|
+
markInventorySubmittedFromOutcomes(target, topic, workType) {
|
|
356
|
+
if (!(0, inventory_1.inventoryPolicy)(target).enabled)
|
|
357
|
+
return;
|
|
358
|
+
for (const outcome of this.outcomes) {
|
|
359
|
+
if (outcome.kind === 'delivery_pending' || outcome.kind === 'submitted' || outcome.kind === 'stored') {
|
|
360
|
+
(0, inventory_1.markInventoryAttempt)(this.database, target, topic, outcome.workId, workType, 'submitted');
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
}
|
|
293
364
|
resolveTopicDay(target) {
|
|
294
365
|
return target.date === 'TODAY'
|
|
295
366
|
? (0, pixiv_date_utils_1.getTodayDate)()
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { TargetConfig } from '../config';
|
|
2
|
+
import type { IDatabase } from '../interfaces/IDatabase';
|
|
3
|
+
import type { CandidateSupplyReport } from '../scheduler/TargetOutcome';
|
|
4
|
+
/** Defaults and bounds for the optional Phase 5 inventory. */
|
|
5
|
+
export interface InventoryPolicy {
|
|
6
|
+
enabled: boolean;
|
|
7
|
+
maxAgeDays: number;
|
|
8
|
+
reserveSize: number;
|
|
9
|
+
fallback: boolean;
|
|
10
|
+
}
|
|
11
|
+
export declare function inventoryPolicy(target: TargetConfig): InventoryPolicy;
|
|
12
|
+
export declare function inventoryTopic(target: TargetConfig): string;
|
|
13
|
+
export declare function inventoryTargetId(target: TargetConfig): string;
|
|
14
|
+
/**
|
|
15
|
+
* Phase 5: after a topic scan yields eligible works that were NOT delivered by
|
|
16
|
+
* this run's lookback loop, upsert them as idle pending inventory. Works that
|
|
17
|
+
* are already recorded as downloaded/delivered are skipped by the caller
|
|
18
|
+
* upstream (selectWorks already removed download-history rows); this method is
|
|
19
|
+
* deliberately idempotent.
|
|
20
|
+
*/
|
|
21
|
+
export declare function recordInventoryCandidates(database: IDatabase, target: TargetConfig, topic: string, date: string, works: Array<{
|
|
22
|
+
id: number;
|
|
23
|
+
type?: string;
|
|
24
|
+
}>, workType: 'illustration' | 'novel'): void;
|
|
25
|
+
/** Enrich a candidate report with the durable pending-count snapshot. */
|
|
26
|
+
export declare function attachInventoryReport(database: IDatabase, target: TargetConfig, topic: string, report: CandidateSupplyReport): CandidateSupplyReport;
|
|
27
|
+
/** Marks a claimed inventory row after a delivery attempt. */
|
|
28
|
+
export declare function markInventoryAttempt(database: IDatabase, target: TargetConfig, topic: string, workId: string, workType: 'illustration' | 'novel', outcome: 'submitted' | 'filtered' | 'pending'): void;
|
|
29
|
+
//# sourceMappingURL=inventory.d.ts.map
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.inventoryPolicy = inventoryPolicy;
|
|
4
|
+
exports.inventoryTopic = inventoryTopic;
|
|
5
|
+
exports.inventoryTargetId = inventoryTargetId;
|
|
6
|
+
exports.recordInventoryCandidates = recordInventoryCandidates;
|
|
7
|
+
exports.attachInventoryReport = attachInventoryReport;
|
|
8
|
+
exports.markInventoryAttempt = markInventoryAttempt;
|
|
9
|
+
function inventoryPolicy(target) {
|
|
10
|
+
const cfg = target.topicProfile?.inventory;
|
|
11
|
+
return {
|
|
12
|
+
enabled: cfg?.enabled === true,
|
|
13
|
+
maxAgeDays: cfg?.maxAgeDays ?? 30,
|
|
14
|
+
reserveSize: Math.min(Math.max(cfg?.reserveSize ?? 20, 1), 100),
|
|
15
|
+
fallback: cfg?.fallback ?? true,
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
function inventoryTopic(target) {
|
|
19
|
+
return (target.topicProfile?.primary?.[0] ?? target.topic ?? '').trim();
|
|
20
|
+
}
|
|
21
|
+
function inventoryTargetId(target) {
|
|
22
|
+
return target.id ?? inventoryTopic(target);
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Phase 5: after a topic scan yields eligible works that were NOT delivered by
|
|
26
|
+
* this run's lookback loop, upsert them as idle pending inventory. Works that
|
|
27
|
+
* are already recorded as downloaded/delivered are skipped by the caller
|
|
28
|
+
* upstream (selectWorks already removed download-history rows); this method is
|
|
29
|
+
* deliberately idempotent.
|
|
30
|
+
*/
|
|
31
|
+
function recordInventoryCandidates(database, target, topic, date, works, workType) {
|
|
32
|
+
const policy = inventoryPolicy(target);
|
|
33
|
+
if (!policy.enabled)
|
|
34
|
+
return;
|
|
35
|
+
const targetId = inventoryTargetId(target);
|
|
36
|
+
for (const work of works) {
|
|
37
|
+
const id = String(work?.id);
|
|
38
|
+
if (!id)
|
|
39
|
+
continue;
|
|
40
|
+
database.candidateInventory.upsert({
|
|
41
|
+
pixivId: id,
|
|
42
|
+
workType,
|
|
43
|
+
topic,
|
|
44
|
+
targetId,
|
|
45
|
+
snapshot: work,
|
|
46
|
+
date,
|
|
47
|
+
maxAgeDays: policy.maxAgeDays,
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
/** Enrich a candidate report with the durable pending-count snapshot. */
|
|
52
|
+
function attachInventoryReport(database, target, topic, report) {
|
|
53
|
+
const policy = inventoryPolicy(target);
|
|
54
|
+
if (!policy.enabled)
|
|
55
|
+
return report;
|
|
56
|
+
const summary = database.candidateInventory.pendingSummary(topic, inventoryTargetId(target));
|
|
57
|
+
return {
|
|
58
|
+
...report,
|
|
59
|
+
inventory: {
|
|
60
|
+
pendingCount: summary.count,
|
|
61
|
+
reserveSize: policy.reserveSize,
|
|
62
|
+
maxAgeDays: policy.maxAgeDays,
|
|
63
|
+
oldestSeenDate: summary.oldestSeenDate,
|
|
64
|
+
},
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
/** Marks a claimed inventory row after a delivery attempt. */
|
|
68
|
+
function markInventoryAttempt(database, target, topic, workId, workType, outcome) {
|
|
69
|
+
if (!inventoryPolicy(target).enabled)
|
|
70
|
+
return;
|
|
71
|
+
const targetId = inventoryTargetId(target);
|
|
72
|
+
const repo = database.candidateInventory;
|
|
73
|
+
if (outcome === 'submitted')
|
|
74
|
+
repo.markSubmitted(workId, workType, topic, targetId);
|
|
75
|
+
else if (outcome === 'filtered')
|
|
76
|
+
repo.markFiltered(workId, workType, topic, targetId);
|
|
77
|
+
else
|
|
78
|
+
repo.markSelectedBackToPending(workId, workType, topic, targetId);
|
|
79
|
+
}
|
|
80
|
+
//# sourceMappingURL=inventory.js.map
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { AccessTokenStore, DownloadRecordInput, ExecutionStatus, SchedulerExecutionRecord } from '../storage/Database';
|
|
2
|
+
import { CandidateInventoryRepository } from '../storage/repositories/CandidateInventoryRepository';
|
|
2
3
|
/**
|
|
3
4
|
* Interface for database operations
|
|
4
5
|
* Provides abstraction for data persistence
|
|
@@ -123,5 +124,7 @@ export interface IDatabase {
|
|
|
123
124
|
};
|
|
124
125
|
countSince(botId: string | null, hours: number): number;
|
|
125
126
|
};
|
|
127
|
+
/** Phase 5 durable CandidateInventory (待发池). */
|
|
128
|
+
readonly candidateInventory: CandidateInventoryRepository;
|
|
126
129
|
}
|
|
127
130
|
//# sourceMappingURL=IDatabase.d.ts.map
|
package/dist/package.json
CHANGED
|
@@ -109,6 +109,17 @@ export interface CandidateSupplyReason {
|
|
|
109
109
|
code: string;
|
|
110
110
|
count: number;
|
|
111
111
|
}
|
|
112
|
+
/** Phase 5: durable 待发池 snapshot for sparse topics (only when enabled). */
|
|
113
|
+
export interface CandidateInventoryReport {
|
|
114
|
+
/** Rows still usable for fallback at report time. */
|
|
115
|
+
pendingCount: number;
|
|
116
|
+
/** Target's configured max retained rows. */
|
|
117
|
+
reserveSize: number;
|
|
118
|
+
/** Max age in days before a pending candidate expires. */
|
|
119
|
+
maxAgeDays: number;
|
|
120
|
+
/** Earliest still-pending first-seen date (YYYY-MM-DD), null when empty. */
|
|
121
|
+
oldestSeenDate: string | null;
|
|
122
|
+
}
|
|
112
123
|
/** Candidate-supply observability snapshot (Phase 1 Candidate Report). */
|
|
113
124
|
export interface CandidateSupplyReport {
|
|
114
125
|
/** Total works surfaced by the topic search before any filtering. */
|
|
@@ -119,6 +130,8 @@ export interface CandidateSupplyReport {
|
|
|
119
130
|
rejected: number;
|
|
120
131
|
/** Why candidates were rejected, by reason code (extensible). */
|
|
121
132
|
reasons: CandidateSupplyReason[];
|
|
133
|
+
/** Phase 5: durable 待发池 reserve (optional, present only when enabled). */
|
|
134
|
+
inventory?: CandidateInventoryReport;
|
|
122
135
|
}
|
|
123
136
|
/**
|
|
124
137
|
* A freshly-harvested upstream funnel with no candidates selected yet.
|
|
@@ -20,7 +20,7 @@ exports.terminalReasonFor = terminalReasonFor;
|
|
|
20
20
|
* A freshly-harvested upstream funnel with no candidates selected yet.
|
|
21
21
|
*/
|
|
22
22
|
function emptyCandidateSupplyReport() {
|
|
23
|
-
return { fetched: 0, selected: 0, rejected: 0, reasons: [] };
|
|
23
|
+
return { fetched: 0, selected: 0, rejected: 0, reasons: [], inventory: undefined };
|
|
24
24
|
}
|
|
25
25
|
/**
|
|
26
26
|
* Fold two supply snapshots of the SAME logical target together, so a
|
|
@@ -43,6 +43,7 @@ function mergeCandidateSupplyReports(first, second) {
|
|
|
43
43
|
selected: first.selected + second.selected,
|
|
44
44
|
rejected: Math.max(0, first.fetched + second.fetched - (first.selected + second.selected)),
|
|
45
45
|
reasons,
|
|
46
|
+
inventory: second.inventory ?? first.inventory,
|
|
46
47
|
};
|
|
47
48
|
}
|
|
48
49
|
/**
|
|
@@ -68,6 +69,7 @@ function withScanSkips(report, scan) {
|
|
|
68
69
|
selected,
|
|
69
70
|
rejected: Math.max(0, base.fetched - selected),
|
|
70
71
|
reasons,
|
|
72
|
+
inventory: base.inventory,
|
|
71
73
|
};
|
|
72
74
|
}
|
|
73
75
|
/**
|
|
@@ -5,6 +5,7 @@ import { OutboxRepository } from './repositories/OutboxRepository';
|
|
|
5
5
|
import { MetadataRepository } from './repositories/MetadataRepository';
|
|
6
6
|
import { SQLiteRateLimitStateStore } from './repositories/RateLimitStateRepository';
|
|
7
7
|
import { SystemErrorRepository } from './repositories/SystemErrorRepository';
|
|
8
|
+
import { CandidateInventoryRepository } from './repositories/CandidateInventoryRepository';
|
|
8
9
|
export interface AccessTokenStore {
|
|
9
10
|
accessToken: string;
|
|
10
11
|
expiresAt: number;
|
|
@@ -48,6 +49,7 @@ export declare class Database implements IDatabase {
|
|
|
48
49
|
private metadataRepo;
|
|
49
50
|
private rateLimitStateStore;
|
|
50
51
|
private systemErrorRepo;
|
|
52
|
+
private candidateInventoryRepo;
|
|
51
53
|
constructor(databasePath: string);
|
|
52
54
|
migrate(): void;
|
|
53
55
|
/** Absolute path of the SQLite file (used to locate sibling cache dirs). */
|
|
@@ -64,6 +66,8 @@ export declare class Database implements IDatabase {
|
|
|
64
66
|
get rateLimitState(): SQLiteRateLimitStateStore;
|
|
65
67
|
/** Durable system-error ledger (observability; never throws). */
|
|
66
68
|
get systemErrors(): SystemErrorRepository;
|
|
69
|
+
/** Phase 5 CandidateInventory (待发池) for sparse topics. */
|
|
70
|
+
get candidateInventory(): CandidateInventoryRepository;
|
|
67
71
|
/** Raw transactional boundary for atomic multi-table intents. */
|
|
68
72
|
transaction<T>(fn: () => T): T;
|
|
69
73
|
/** Expose a prepared-statement helper if needed by services (pragmas etc). */
|
package/dist/storage/Database.js
CHANGED
|
@@ -18,6 +18,7 @@ const OutboxRepository_1 = require("./repositories/OutboxRepository");
|
|
|
18
18
|
const MetadataRepository_1 = require("./repositories/MetadataRepository");
|
|
19
19
|
const RateLimitStateRepository_1 = require("./repositories/RateLimitStateRepository");
|
|
20
20
|
const SystemErrorRepository_1 = require("./repositories/SystemErrorRepository");
|
|
21
|
+
const CandidateInventoryRepository_1 = require("./repositories/CandidateInventoryRepository");
|
|
21
22
|
const NodeSqliteDriver_1 = require("./drivers/NodeSqliteDriver");
|
|
22
23
|
class Database {
|
|
23
24
|
databasePath;
|
|
@@ -35,6 +36,7 @@ class Database {
|
|
|
35
36
|
metadataRepo;
|
|
36
37
|
rateLimitStateStore;
|
|
37
38
|
systemErrorRepo;
|
|
39
|
+
candidateInventoryRepo;
|
|
38
40
|
constructor(databasePath) {
|
|
39
41
|
this.databasePath = databasePath;
|
|
40
42
|
try {
|
|
@@ -63,6 +65,7 @@ class Database {
|
|
|
63
65
|
this.metadataRepo = new MetadataRepository_1.MetadataRepository(this.db);
|
|
64
66
|
this.rateLimitStateStore = new RateLimitStateRepository_1.SQLiteRateLimitStateStore(this.db);
|
|
65
67
|
this.systemErrorRepo = new SystemErrorRepository_1.SystemErrorRepository(this.db);
|
|
68
|
+
this.candidateInventoryRepo = new CandidateInventoryRepository_1.CandidateInventoryRepository(this.db);
|
|
66
69
|
}
|
|
67
70
|
catch (error) {
|
|
68
71
|
throw new errors_1.DatabaseError(`Failed to initialize database at ${this.databasePath}`, error instanceof Error ? error : undefined);
|
|
@@ -102,6 +105,10 @@ class Database {
|
|
|
102
105
|
get systemErrors() {
|
|
103
106
|
return this.systemErrorRepo;
|
|
104
107
|
}
|
|
108
|
+
/** Phase 5 CandidateInventory (待发池) for sparse topics. */
|
|
109
|
+
get candidateInventory() {
|
|
110
|
+
return this.candidateInventoryRepo;
|
|
111
|
+
}
|
|
105
112
|
/** Raw transactional boundary for atomic multi-table intents. */
|
|
106
113
|
transaction(fn) {
|
|
107
114
|
return this.db.transaction(fn)();
|
|
@@ -222,6 +222,25 @@ class DatabaseMigration {
|
|
|
222
222
|
scope TEXT PRIMARY KEY,
|
|
223
223
|
state TEXT NOT NULL,
|
|
224
224
|
updated_at INTEGER NOT NULL
|
|
225
|
+
)`,
|
|
226
|
+
// Phase 5 CandidateInventory: durable 待发池 for sparse topics. Same
|
|
227
|
+
// database and transaction world as the Slot Ledger; never a second
|
|
228
|
+
// state authority. Only populated when target.topicProfile.inventory.enabled.
|
|
229
|
+
`CREATE TABLE IF NOT EXISTS candidate_inventory (
|
|
230
|
+
pixiv_id TEXT NOT NULL,
|
|
231
|
+
work_type TEXT NOT NULL,
|
|
232
|
+
topic TEXT NOT NULL,
|
|
233
|
+
target_id TEXT NOT NULL,
|
|
234
|
+
status TEXT NOT NULL DEFAULT 'pending',
|
|
235
|
+
snapshot_json TEXT NOT NULL,
|
|
236
|
+
first_seen_date TEXT NOT NULL,
|
|
237
|
+
last_seen_date TEXT NOT NULL,
|
|
238
|
+
seen_count INTEGER NOT NULL DEFAULT 1,
|
|
239
|
+
attempt_count INTEGER NOT NULL DEFAULT 0,
|
|
240
|
+
expires_at TEXT NOT NULL,
|
|
241
|
+
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
242
|
+
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
243
|
+
PRIMARY KEY (pixiv_id, work_type, topic, target_id)
|
|
225
244
|
)`,
|
|
226
245
|
// Durable error event ledger for observability (structured error
|
|
227
246
|
// taxonomy; populated by download/system handlers and shown in the
|
|
@@ -244,6 +263,7 @@ class DatabaseMigration {
|
|
|
244
263
|
resolved_at DATETIME
|
|
245
264
|
)`,
|
|
246
265
|
`CREATE INDEX IF NOT EXISTS idx_system_errors_bot_created ON system_errors(bot_id, created_at)`,
|
|
266
|
+
`CREATE INDEX IF NOT EXISTS idx_cinventory_pending ON candidate_inventory(status, first_seen_date, expires_at)`,
|
|
247
267
|
];
|
|
248
268
|
// Phase 1: create tables (idempotent). Must run before any PRAGMA-based
|
|
249
269
|
// column check, otherwise a fresh DB would report the table as missing and
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { BaseRepository } from './BaseRepository';
|
|
2
|
+
export type CandidateInventoryStatus = 'pending' | 'selected' | 'submitted' | 'filtered' | 'expired';
|
|
3
|
+
export interface CandidateInventoryRow {
|
|
4
|
+
pixivId: string;
|
|
5
|
+
workType: 'illustration' | 'novel';
|
|
6
|
+
topic: string;
|
|
7
|
+
targetId: string;
|
|
8
|
+
status: CandidateInventoryStatus;
|
|
9
|
+
snapshotJson: string;
|
|
10
|
+
firstSeenDate: string;
|
|
11
|
+
lastSeenDate: string;
|
|
12
|
+
seenCount: number;
|
|
13
|
+
attemptCount: number;
|
|
14
|
+
expiresAt: string;
|
|
15
|
+
createdAt: string;
|
|
16
|
+
updatedAt: string;
|
|
17
|
+
}
|
|
18
|
+
export interface CandidateInventorySnapshot {
|
|
19
|
+
pixivId: string;
|
|
20
|
+
workType: 'illustration' | 'novel';
|
|
21
|
+
topic: string;
|
|
22
|
+
targetId: string;
|
|
23
|
+
snapshot: unknown;
|
|
24
|
+
date: string;
|
|
25
|
+
maxAgeDays: number;
|
|
26
|
+
}
|
|
27
|
+
/** Phase 5 durable 待发池 — same SQLite database as the Slot Ledger. */
|
|
28
|
+
export declare class CandidateInventoryRepository extends BaseRepository {
|
|
29
|
+
upsert(input: CandidateInventorySnapshot): void;
|
|
30
|
+
/** Marks a claimed candidate and returns its snapshot for the pipeline. */
|
|
31
|
+
claimNext(input: {
|
|
32
|
+
topic: string;
|
|
33
|
+
targetId: string;
|
|
34
|
+
reserveSize: number;
|
|
35
|
+
date: string;
|
|
36
|
+
}): CandidateInventoryRow | null;
|
|
37
|
+
markSubmitted(pixivId: string, workType: string, topic: string, targetId: string): void;
|
|
38
|
+
markFiltered(pixivId: string, workType: string, topic: string, targetId: string): void;
|
|
39
|
+
markSelectedBackToPending(pixivId: string, workType: string, topic: string, targetId: string): void;
|
|
40
|
+
countPending(topic: string, targetId: string): number;
|
|
41
|
+
pendingSummary(topic: string, targetId: string): {
|
|
42
|
+
count: number;
|
|
43
|
+
oldestSeenDate: string | null;
|
|
44
|
+
};
|
|
45
|
+
/** Sweep rows past maxAgeDays; idempotent per scheduled scan. */
|
|
46
|
+
evictExpired(topic: string, targetId: string): number;
|
|
47
|
+
private toRow;
|
|
48
|
+
private todayUtc;
|
|
49
|
+
private addDaysUtc;
|
|
50
|
+
}
|
|
51
|
+
//# sourceMappingURL=CandidateInventoryRepository.d.ts.map
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.CandidateInventoryRepository = void 0;
|
|
4
|
+
const BaseRepository_1 = require("./BaseRepository");
|
|
5
|
+
/** Phase 5 durable 待发池 — same SQLite database as the Slot Ledger. */
|
|
6
|
+
class CandidateInventoryRepository extends BaseRepository_1.BaseRepository {
|
|
7
|
+
upsert(input) {
|
|
8
|
+
const expiresAt = this.addDaysUtc(input.date, input.maxAgeDays);
|
|
9
|
+
this.db
|
|
10
|
+
.prepare(`INSERT INTO candidate_inventory
|
|
11
|
+
(pixiv_id, work_type, topic, target_id, status, snapshot_json,
|
|
12
|
+
first_seen_date, last_seen_date, seen_count, attempt_count, expires_at)
|
|
13
|
+
VALUES
|
|
14
|
+
(@pixivId, @workType, @topic, @targetId, 'pending', @snapshot,
|
|
15
|
+
@date, @date, 1, 0, @expiresAt)
|
|
16
|
+
ON CONFLICT(pixiv_id, work_type, topic, target_id) DO UPDATE SET
|
|
17
|
+
snapshot_json = excluded.snapshot_json,
|
|
18
|
+
last_seen_date = excluded.last_seen_date,
|
|
19
|
+
seen_count = candidate_inventory.seen_count + 1,
|
|
20
|
+
status = CASE
|
|
21
|
+
WHEN candidate_inventory.status IN ('selected','submitted','filtered','expired')
|
|
22
|
+
THEN candidate_inventory.status
|
|
23
|
+
ELSE 'pending'
|
|
24
|
+
END,
|
|
25
|
+
expires_at = excluded.expires_at,
|
|
26
|
+
updated_at = CURRENT_TIMESTAMP`)
|
|
27
|
+
.run({
|
|
28
|
+
pixivId: input.pixivId,
|
|
29
|
+
workType: input.workType,
|
|
30
|
+
topic: input.topic,
|
|
31
|
+
targetId: input.targetId,
|
|
32
|
+
snapshot: JSON.stringify(input.snapshot),
|
|
33
|
+
date: input.date,
|
|
34
|
+
expiresAt,
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
/** Marks a claimed candidate and returns its snapshot for the pipeline. */
|
|
38
|
+
claimNext(input) {
|
|
39
|
+
const rows = this.db
|
|
40
|
+
.prepare(`SELECT * FROM candidate_inventory
|
|
41
|
+
WHERE topic = ? AND target_id = ?
|
|
42
|
+
AND status = 'pending'
|
|
43
|
+
AND expires_at >= ?
|
|
44
|
+
ORDER BY first_seen_date ASC, seen_count ASC
|
|
45
|
+
LIMIT ?`)
|
|
46
|
+
.all(input.topic, input.targetId, this.addDaysUtc(input.date, 0), input.reserveSize);
|
|
47
|
+
for (const row of rows) {
|
|
48
|
+
const updated = this.db
|
|
49
|
+
.prepare(`UPDATE candidate_inventory
|
|
50
|
+
SET status = 'selected', attempt_count = attempt_count + 1, updated_at = CURRENT_TIMESTAMP
|
|
51
|
+
WHERE pixiv_id = ? AND work_type = ? AND topic = ? AND target_id = ? AND status = 'pending'`)
|
|
52
|
+
.run(row.pixiv_id, row.work_type, row.topic, row.target_id);
|
|
53
|
+
if (updated.changes > 0)
|
|
54
|
+
return this.toRow(row);
|
|
55
|
+
}
|
|
56
|
+
return null;
|
|
57
|
+
}
|
|
58
|
+
markSubmitted(pixivId, workType, topic, targetId) {
|
|
59
|
+
this.db
|
|
60
|
+
.prepare(`UPDATE candidate_inventory
|
|
61
|
+
SET status = 'submitted', updated_at = CURRENT_TIMESTAMP
|
|
62
|
+
WHERE pixiv_id = ? AND work_type = ? AND topic = ? AND target_id = ?`)
|
|
63
|
+
.run(pixivId, workType, topic, targetId);
|
|
64
|
+
}
|
|
65
|
+
markFiltered(pixivId, workType, topic, targetId) {
|
|
66
|
+
this.db
|
|
67
|
+
.prepare(`UPDATE candidate_inventory
|
|
68
|
+
SET status = 'filtered', updated_at = CURRENT_TIMESTAMP
|
|
69
|
+
WHERE pixiv_id = ? AND work_type = ? AND topic = ? AND target_id = ?`)
|
|
70
|
+
.run(pixivId, workType, topic, targetId);
|
|
71
|
+
}
|
|
72
|
+
markSelectedBackToPending(pixivId, workType, topic, targetId) {
|
|
73
|
+
this.db
|
|
74
|
+
.prepare(`UPDATE candidate_inventory
|
|
75
|
+
SET status = 'pending', updated_at = CURRENT_TIMESTAMP
|
|
76
|
+
WHERE pixiv_id = ? AND work_type = ? AND topic = ? AND target_id = ? AND status = 'selected'`)
|
|
77
|
+
.run(pixivId, workType, topic, targetId);
|
|
78
|
+
}
|
|
79
|
+
countPending(topic, targetId) {
|
|
80
|
+
const rows = this.db
|
|
81
|
+
.prepare(`SELECT COUNT(*) AS n FROM candidate_inventory WHERE topic = ? AND target_id = ? AND status = 'pending' AND expires_at >= ?`)
|
|
82
|
+
.get(topic, targetId, this.todayUtc());
|
|
83
|
+
return Number(rows?.n ?? 0);
|
|
84
|
+
}
|
|
85
|
+
pendingSummary(topic, targetId) {
|
|
86
|
+
const row = this.db
|
|
87
|
+
.prepare(`SELECT COUNT(*) AS n, MIN(first_seen_date) AS oldest
|
|
88
|
+
FROM candidate_inventory
|
|
89
|
+
WHERE topic = ? AND target_id = ? AND status = 'pending' AND expires_at >= ?`)
|
|
90
|
+
.get(topic, targetId, this.todayUtc());
|
|
91
|
+
return { count: Number(row?.n ?? 0), oldestSeenDate: row?.oldest ?? null };
|
|
92
|
+
}
|
|
93
|
+
/** Sweep rows past maxAgeDays; idempotent per scheduled scan. */
|
|
94
|
+
evictExpired(topic, targetId) {
|
|
95
|
+
const info = this.db
|
|
96
|
+
.prepare(`UPDATE candidate_inventory
|
|
97
|
+
SET status = 'expired', updated_at = CURRENT_TIMESTAMP
|
|
98
|
+
WHERE topic = ? AND target_id = ? AND status IN ('pending','selected') AND expires_at < ?`)
|
|
99
|
+
.run(topic, targetId, this.todayUtc());
|
|
100
|
+
return info.changes;
|
|
101
|
+
}
|
|
102
|
+
toRow(row) {
|
|
103
|
+
return {
|
|
104
|
+
pixivId: String(row.pixiv_id),
|
|
105
|
+
workType: row.work_type,
|
|
106
|
+
topic: row.topic,
|
|
107
|
+
targetId: row.target_id,
|
|
108
|
+
status: row.status,
|
|
109
|
+
snapshotJson: row.snapshot_json,
|
|
110
|
+
firstSeenDate: row.first_seen_date,
|
|
111
|
+
lastSeenDate: row.last_seen_date,
|
|
112
|
+
seenCount: Number(row.seen_count ?? 1),
|
|
113
|
+
attemptCount: Number(row.attempt_count ?? 0),
|
|
114
|
+
expiresAt: row.expires_at,
|
|
115
|
+
createdAt: row.created_at,
|
|
116
|
+
updatedAt: row.updated_at,
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
todayUtc() {
|
|
120
|
+
return new Date().toISOString().slice(0, 10);
|
|
121
|
+
}
|
|
122
|
+
addDaysUtc(date, days) {
|
|
123
|
+
const d = new Date(`${date}T00:00:00.000Z`);
|
|
124
|
+
d.setUTCDate(d.getUTCDate() + days);
|
|
125
|
+
return d.toISOString().slice(0, 10);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
exports.CandidateInventoryRepository = CandidateInventoryRepository;
|
|
129
|
+
//# sourceMappingURL=CandidateInventoryRepository.js.map
|
package/dist/version.js
CHANGED
|
@@ -2,5 +2,5 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.BUILD = void 0;
|
|
4
4
|
// GENERATED by scripts/write-version.js — do not edit manually.
|
|
5
|
-
exports.BUILD = { version: '2.
|
|
5
|
+
exports.BUILD = { version: '2.30.0', commit: '153116bd9b08' };
|
|
6
6
|
//# sourceMappingURL=version.js.map
|
package/dist/webui/package.json
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pixivflow",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.30.0",
|
|
4
4
|
"description": "🎨 Pixiv 下载、筛选与自动收集工具 - 批量下载插画和小说、按标签/热度/日期筛选、定时任务与可靠 HTTP 交付 | Pixiv downloader and automation toolkit with filtering, scheduling and reliable HTTP delivery",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|