pixivflow 2.29.0 → 2.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +2 -2
- package/README.md +2 -2
- package/dist/commands/WebUICommand.d.ts +2 -2
- package/dist/commands/WebUICommand.js +9 -7
- package/dist/config/types.d.ts +31 -0
- package/dist/config/validation.js +35 -0
- package/dist/download/handlers/IllustrationTargetHandler.d.ts +8 -0
- package/dist/download/handlers/IllustrationTargetHandler.js +74 -0
- package/dist/download/handlers/NovelTargetHandler.d.ts +7 -0
- package/dist/download/handlers/NovelTargetHandler.js +71 -0
- package/dist/download/inventory.d.ts +29 -0
- package/dist/download/inventory.js +80 -0
- package/dist/interfaces/IDatabase.d.ts +3 -0
- package/dist/package.json +1 -1
- package/dist/scheduler/TargetOutcome.d.ts +13 -0
- package/dist/scheduler/TargetOutcome.js +3 -1
- package/dist/storage/Database.d.ts +4 -0
- package/dist/storage/Database.js +7 -0
- package/dist/storage/DatabaseMigration.js +20 -0
- package/dist/storage/repositories/CandidateInventoryRepository.d.ts +51 -0
- package/dist/storage/repositories/CandidateInventoryRepository.js +129 -0
- package/dist/version.js +1 -1
- package/dist/webui/package.json +1 -1
- package/package.json +2 -1
- package/webui-frontend/dist/assets/CheckCircleOutlined-H3DnUyMO.js +2 -0
- package/webui-frontend/dist/assets/ClearOutlined-BpODq7I7.js +2 -0
- package/webui-frontend/dist/assets/CloseCircleOutlined-BFfS5iXd.js +2 -0
- package/webui-frontend/dist/assets/Config-DRElCeDC.js +16 -0
- package/webui-frontend/dist/assets/Dashboard-aqPMquSO.js +2 -0
- package/webui-frontend/dist/assets/DeleteOutlined-BFmW1oWe.js +2 -0
- package/webui-frontend/dist/assets/Download-DCKDU51K.js +2 -0
- package/webui-frontend/dist/assets/Files-nNoRY5tl.js +5 -0
- package/webui-frontend/dist/assets/PictureOutlined-U51jGr2U.js +2 -0
- package/webui-frontend/dist/assets/SortDescendingOutlined-CckVuL_b.js +2 -0
- package/webui-frontend/dist/assets/ThunderboltOutlined-ulcCd7ui.js +2 -0
- package/webui-frontend/dist/assets/UrlDownload-Bzs0AZN0.js +3 -0
- package/webui-frontend/dist/assets/dateUtils-gQt2uAjS.js +2 -0
- package/webui-frontend/dist/assets/errorCodeTranslator-BqIackMm.js +3 -0
- package/webui-frontend/dist/assets/index-B2ns27ii.js +2 -0
- package/webui-frontend/dist/assets/index-B7yYDv10.js +2 -0
- package/webui-frontend/dist/assets/index-Bbsa44oT.js +2 -0
- package/webui-frontend/dist/assets/index-C2ANILg9.js +19 -0
- package/webui-frontend/dist/assets/index-CDrp3NIf.js +8 -0
- package/webui-frontend/dist/assets/index-CFtP5lqo.js +22 -0
- package/webui-frontend/dist/assets/index-CJln3zA7.js +14 -0
- package/webui-frontend/dist/assets/index-CZhfhXvc.js +2 -0
- package/webui-frontend/dist/assets/index-CfHs6AUn.js +2 -0
- package/webui-frontend/dist/assets/index-DG6kGbYq.js +217 -0
- package/webui-frontend/dist/assets/index-DHwJxrFi.css +1 -0
- package/webui-frontend/dist/assets/index-DPigBPWX.js +2 -0
- package/webui-frontend/dist/assets/index-DarObP-C.js +2 -0
- package/webui-frontend/dist/assets/index-_EG4kaKN.js +47 -0
- package/webui-frontend/dist/assets/row-_c51T55h.js +20 -0
- package/webui-frontend/dist/assets/socket-oqYz9cKb.js +2 -0
- package/webui-frontend/dist/assets/useAuth-wCjAOx6S.js +9 -0
- package/webui-frontend/dist/assets/useConfig-Ds7RpIL1.js +9 -0
- package/webui-frontend/dist/assets/useDownload-BlxvZNEm.js +2 -0
- package/webui-frontend/dist/assets/useErrorHandler-BOvHue1w.js +84 -0
- package/webui-frontend/dist/index.html +15 -0
package/README.en.md
CHANGED
|
@@ -69,7 +69,7 @@ pixivflow scheduler # long-running cron collection
|
|
|
69
69
|
```
|
|
70
70
|
|
|
71
71
|
Prefer not to hand-write config? The interactive wizard `pixivflow setup` generates it.
|
|
72
|
-
For a GUI, run `pixivflow
|
|
72
|
+
For a GUI, run `pixivflow web` (frontend:
|
|
73
73
|
[pixivflow-webui](https://github.com/redtidev1918/pixivflow-webui)).
|
|
74
74
|
|
|
75
75
|
Downloading Pixiv ugoira additionally needs `python3` and `ffmpeg`: frames are
|
|
@@ -249,7 +249,7 @@ ledger; dry-run by default).
|
|
|
249
249
|
| `pixivflow download --url <url>` | Direct download via URL |
|
|
250
250
|
| `pixivflow random` | Random popular artwork |
|
|
251
251
|
| `pixivflow scheduler` | Start scheduled jobs |
|
|
252
|
-
| `pixivflow
|
|
252
|
+
| `pixivflow web` | Start the WebUI |
|
|
253
253
|
| `pixivflow config` | Manage config (view / edit / backup / restore) |
|
|
254
254
|
| `pixivflow status` | Download stats and recent records |
|
|
255
255
|
| `pixivflow health` | Health check: config, directories, connectivity |
|
package/README.md
CHANGED
|
@@ -64,7 +64,7 @@ pixivflow scheduler # 按 cron 配置长期挂机自动收集
|
|
|
64
64
|
```
|
|
65
65
|
|
|
66
66
|
不想手写配置?运行交互式向导 `pixivflow setup` 一步步生成。图形界面用
|
|
67
|
-
`pixivflow
|
|
67
|
+
`pixivflow web`(前端见 [pixivflow-webui](https://github.com/redtidev1918/pixivflow-webui))。
|
|
68
68
|
|
|
69
69
|
下载 Pixiv 动图(ugoira)还需要 `python3` 和 `ffmpeg`:程序按逐帧延迟合成循环 GIF,
|
|
70
70
|
可直接作为动画交付给下游。官方 Docker 镜像已包含两者,详见
|
|
@@ -227,7 +227,7 @@ pixivflow outbox cancel <id> # 只取消尚未执行的 row
|
|
|
227
227
|
| `pixivflow download --url <url>` | 通过 URL 直接下载 |
|
|
228
228
|
| `pixivflow random` | 随机下载热门作品 |
|
|
229
229
|
| `pixivflow scheduler` | 启动定时任务 |
|
|
230
|
-
| `pixivflow
|
|
230
|
+
| `pixivflow web` | 启动 WebUI |
|
|
231
231
|
| `pixivflow config` | 配置管理(查看 / 编辑 / 备份 / 恢复) |
|
|
232
232
|
| `pixivflow status` | 下载统计与最近记录 |
|
|
233
233
|
| `pixivflow health` | 健康检查:配置、目录可写性、连通性 |
|
|
@@ -8,8 +8,8 @@ import { CommandContext, CommandArgs, CommandResult } from './types';
|
|
|
8
8
|
* WebUI command - Start WebUI server
|
|
9
9
|
*/
|
|
10
10
|
export declare class WebUICommand extends BaseCommand {
|
|
11
|
-
readonly name = "
|
|
12
|
-
readonly description = "Start WebUI server";
|
|
11
|
+
readonly name = "web";
|
|
12
|
+
readonly description = "Start WebUI server (memory only when used)";
|
|
13
13
|
readonly aliases: string[];
|
|
14
14
|
readonly metadata: {
|
|
15
15
|
category: CommandCategory;
|
|
@@ -18,9 +18,9 @@ const child_process_1 = require("child_process");
|
|
|
18
18
|
* WebUI command - Start WebUI server
|
|
19
19
|
*/
|
|
20
20
|
class WebUICommand extends Command_1.BaseCommand {
|
|
21
|
-
name = '
|
|
22
|
-
description = 'Start WebUI server';
|
|
23
|
-
aliases = ['w'];
|
|
21
|
+
name = 'web';
|
|
22
|
+
description = 'Start WebUI server (memory only when used)';
|
|
23
|
+
aliases = ['w', 'webui'];
|
|
24
24
|
metadata = {
|
|
25
25
|
category: metadata_1.CommandCategory.UTILITY,
|
|
26
26
|
requiresAuth: false,
|
|
@@ -437,10 +437,12 @@ Auto-Build Feature:
|
|
|
437
437
|
the command will automatically build the frontend before starting.
|
|
438
438
|
|
|
439
439
|
Examples:
|
|
440
|
-
pixivflow
|
|
441
|
-
pixivflow
|
|
442
|
-
pixivflow
|
|
443
|
-
pixivflow
|
|
440
|
+
pixivflow web # Start WebUI (loads frontend only now)
|
|
441
|
+
pixivflow web --port 8080 # Use custom port
|
|
442
|
+
pixivflow web --host 0.0.0.0 # Bind to all interfaces
|
|
443
|
+
pixivflow web --static-path ./webui-frontend/dist # Use custom static path
|
|
444
|
+
|
|
445
|
+
Backward-compatible aliases: pixivflow webui, pixivflow w`;
|
|
444
446
|
}
|
|
445
447
|
}
|
|
446
448
|
exports.WebUICommand = WebUICommand;
|
package/dist/config/types.d.ts
CHANGED
|
@@ -27,6 +27,35 @@ export interface CandidateCollectionConfig {
|
|
|
27
27
|
/** Minimum metadata-topic score to survive filtering (default 0.35). */
|
|
28
28
|
minMetadataScore?: number;
|
|
29
29
|
}
|
|
30
|
+
/** Phase 4: topic-supply strategy (per-target, optional, default off). */
|
|
31
|
+
export interface TopicProfileStrategyConfig {
|
|
32
|
+
/** Relative weight for publishing fresher works first (0..1, default 0.4). */
|
|
33
|
+
freshnessWeight?: number;
|
|
34
|
+
/** Relative weight for publishing higher-popularity works first (0..1, default 0.3). */
|
|
35
|
+
popularityWeight?: number;
|
|
36
|
+
}
|
|
37
|
+
/** Phase 5: durable candidate inventory ("待发池") for a target. */
|
|
38
|
+
export interface CandidateInventoryConfig {
|
|
39
|
+
/** Master switch. Default false: no collection, no fallback, no new columns read. */
|
|
40
|
+
enabled?: boolean;
|
|
41
|
+
/** Max age (days) a pending candidate may stay usable for fallback. Default 30. */
|
|
42
|
+
maxAgeDays?: number;
|
|
43
|
+
/** Max pending rows retained per (topic,target). Default 20. */
|
|
44
|
+
reserveSize?: number;
|
|
45
|
+
/** When the fresh/lookback scan is empty, publish oldest pending inventory. Default true. */
|
|
46
|
+
fallback?: boolean;
|
|
47
|
+
}
|
|
48
|
+
/** Phase 4: TopicProfile (seed + related tags + supply strategy). */
|
|
49
|
+
export interface TopicProfileConfig {
|
|
50
|
+
/** Primary seed tag(s); first item is the canonical topic when topic is unset. */
|
|
51
|
+
primary?: string[];
|
|
52
|
+
/** Related tags that strengthen topic membership evidence. */
|
|
53
|
+
related?: string[];
|
|
54
|
+
/** Ranking strategy knobs (Phase 4; wired after CandidateInventory). */
|
|
55
|
+
strategy?: TopicProfileStrategyConfig;
|
|
56
|
+
/** CandidateInventory policy (Phase 5). */
|
|
57
|
+
inventory?: CandidateInventoryConfig;
|
|
58
|
+
}
|
|
30
59
|
/** Behaviour when a target cannot produce the requested number of works. */
|
|
31
60
|
export interface NoMatchPolicyConfig {
|
|
32
61
|
/**
|
|
@@ -178,6 +207,8 @@ export interface TargetConfig {
|
|
|
178
207
|
candidateCollection?: CandidateCollectionConfig;
|
|
179
208
|
/** Bounded fallback and notification policy for an empty result. */
|
|
180
209
|
noMatchPolicy?: NoMatchPolicyConfig;
|
|
210
|
+
/** Phase 4/5: topic supply profile + candidate inventory policy (all optional). */
|
|
211
|
+
topicProfile?: TopicProfileConfig;
|
|
181
212
|
/**
|
|
182
213
|
* Ranking mode (only used when mode='ranking')
|
|
183
214
|
* - 'day': Daily ranking
|
|
@@ -240,6 +240,41 @@ function validateConfig(config, location, databasePath) {
|
|
|
240
240
|
if (target.noMatchPolicy?.notify !== undefined && typeof target.noMatchPolicy.notify !== 'boolean') {
|
|
241
241
|
errors.push(`targets[${index}].noMatchPolicy.notify: Must be a boolean`);
|
|
242
242
|
}
|
|
243
|
+
if (target.topicProfile) {
|
|
244
|
+
const tp = target.topicProfile;
|
|
245
|
+
if (tp.primary !== undefined && (!Array.isArray(tp.primary) || tp.primary.length === 0 || tp.primary.some((x) => typeof x !== 'string' || !x.trim()))) {
|
|
246
|
+
errors.push(`targets[${index}].topicProfile.primary: Must be a non-empty string array`);
|
|
247
|
+
}
|
|
248
|
+
if (tp.related !== undefined && (!Array.isArray(tp.related) || tp.related.some((x) => typeof x !== 'string' || !x.trim()))) {
|
|
249
|
+
errors.push(`targets[${index}].topicProfile.related: Must be a string array`);
|
|
250
|
+
}
|
|
251
|
+
const st = tp.strategy;
|
|
252
|
+
if (st) {
|
|
253
|
+
for (const field of ['freshnessWeight', 'popularityWeight']) {
|
|
254
|
+
const value = st[field];
|
|
255
|
+
if (value !== undefined && (typeof value !== 'number' || value < 0 || value > 1)) {
|
|
256
|
+
errors.push(`targets[${index}].topicProfile.strategy.${field}: Must be a number between 0 and 1`);
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
const inv = tp.inventory;
|
|
261
|
+
if (inv) {
|
|
262
|
+
if (inv.enabled !== undefined && typeof inv.enabled !== 'boolean') {
|
|
263
|
+
errors.push(`targets[${index}].topicProfile.inventory.enabled: Must be a boolean`);
|
|
264
|
+
}
|
|
265
|
+
const bound = (field) => {
|
|
266
|
+
const value = inv[field];
|
|
267
|
+
if (value !== undefined && (typeof value !== 'number' || !Number.isInteger(value) || value < 1 || value > 365)) {
|
|
268
|
+
errors.push(`targets[${index}].topicProfile.inventory.${field}: Must be an integer between 1 and 365`);
|
|
269
|
+
}
|
|
270
|
+
};
|
|
271
|
+
bound('maxAgeDays');
|
|
272
|
+
bound('reserveSize');
|
|
273
|
+
if (inv.fallback !== undefined && typeof inv.fallback !== 'boolean') {
|
|
274
|
+
errors.push(`targets[${index}].topicProfile.inventory.fallback: Must be a boolean`);
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
}
|
|
243
278
|
if (target.noMatchPolicy?.notify === true) {
|
|
244
279
|
const deliveryTarget = target.delivery?.target?.trim();
|
|
245
280
|
const notifyTarget = deliveryTarget ? config.delivery?.targets?.[deliveryTarget] : undefined;
|
|
@@ -68,6 +68,14 @@ export declare class IllustrationTargetHandler {
|
|
|
68
68
|
private fetchIllustrations;
|
|
69
69
|
private fetchTopicIllustrations;
|
|
70
70
|
private handleTopicWithLookback;
|
|
71
|
+
/**
|
|
72
|
+
* Phase 5 fallback: claim oldest pending candidate(s) from the durable
|
|
73
|
+
* 待发池 and try to publish them exactly like a normal day's pipeline run.
|
|
74
|
+
* Returns true when at least one work reached a delivery/stored outcome.
|
|
75
|
+
*/
|
|
76
|
+
private tryInventoryFallback;
|
|
77
|
+
/** Mark inventory rows whose work reached a delivery/stored outcome. */
|
|
78
|
+
private markInventorySubmittedFromOutcomes;
|
|
71
79
|
private resolveTopicDay;
|
|
72
80
|
private shiftDay;
|
|
73
81
|
private fetchRankingIllustrations;
|
|
@@ -12,6 +12,7 @@ const observability_1 = require("../../observability");
|
|
|
12
12
|
const context_1 = require("../../observability/context");
|
|
13
13
|
const DownloadPlanner_1 = require("../plan/DownloadPlanner");
|
|
14
14
|
const deliveryContext_1 = require("./deliveryContext");
|
|
15
|
+
const inventory_1 = require("../inventory");
|
|
15
16
|
class IllustrationTargetHandler {
|
|
16
17
|
client;
|
|
17
18
|
database;
|
|
@@ -233,6 +234,9 @@ class IllustrationTargetHandler {
|
|
|
233
234
|
const pipeline = this.topicPipelineFactory();
|
|
234
235
|
const { works, selection } = await pipeline.selectWorks(target, 'illustration', day, selectionLimit, target.topicDiscovery ?? {}, target.candidateCollection ?? {});
|
|
235
236
|
logger_1.logger.info(`Topic "${topic}" illustration: tags=${selection.resolvedTagCount} raw=${selection.rawCount} deduped=${selection.dedupedCount} aiExcluded=${selection.aiExcludedCount} accepted=${selection.acceptedCount} candidates=${works.length} target=${limit}`);
|
|
237
|
+
// Phase 5: harvest the accepted-but-not-yet-delivered works into the
|
|
238
|
+
// durable 待发池 (idempotent; only when the target opts in).
|
|
239
|
+
(0, inventory_1.recordInventoryCandidates)(this.database, target, topic, day, works, 'illustration');
|
|
236
240
|
// Candidate Supply Observability: fold this lookback day's upstream funnel
|
|
237
241
|
// into the target's accumulated Candidate Report.
|
|
238
242
|
const report = (0, TargetOutcome_1.emptyCandidateSupplyReport)();
|
|
@@ -270,6 +274,7 @@ class IllustrationTargetHandler {
|
|
|
270
274
|
this.scan = (0, TargetOutcome_1.mergeScanSummaries)(this.scan, result.scan);
|
|
271
275
|
if (result.downloaded > 0) {
|
|
272
276
|
this.handleDownloadResult(result, target, 'topic', illusts.length);
|
|
277
|
+
this.markInventorySubmittedFromOutcomes(target, (0, inventory_1.inventoryTopic)(target), 'illustration');
|
|
273
278
|
return;
|
|
274
279
|
}
|
|
275
280
|
if (this.scan && this.scan.outages.length > 0) {
|
|
@@ -278,6 +283,16 @@ class IllustrationTargetHandler {
|
|
|
278
283
|
return;
|
|
279
284
|
}
|
|
280
285
|
}
|
|
286
|
+
// Phase 5: when the fresh + lookback scan produced nothing, and the target
|
|
287
|
+
// opted into CandidateInventory, publish oldest pending reserve candidates.
|
|
288
|
+
const topic = (0, inventory_1.inventoryTopic)(target);
|
|
289
|
+
if (await this.tryInventoryFallback(target, topic, 'illustration')) {
|
|
290
|
+
return;
|
|
291
|
+
}
|
|
292
|
+
// Report how much reserve remains so the empty result is not a dead end.
|
|
293
|
+
if (this.supplyRep) {
|
|
294
|
+
this.supplyRep = (0, inventory_1.attachInventoryReport)(this.database, target, topic, this.supplyRep);
|
|
295
|
+
}
|
|
281
296
|
const scan = this.scan;
|
|
282
297
|
// An explicit no-eligible-candidate verdict needs something to have been
|
|
283
298
|
// considered. When the scan is empty (nothing surfaced at all) the richer
|
|
@@ -297,6 +312,65 @@ class IllustrationTargetHandler {
|
|
|
297
312
|
...(scan ? { scan } : {}),
|
|
298
313
|
});
|
|
299
314
|
}
|
|
315
|
+
/**
|
|
316
|
+
* Phase 5 fallback: claim oldest pending candidate(s) from the durable
|
|
317
|
+
* 待发池 and try to publish them exactly like a normal day's pipeline run.
|
|
318
|
+
* Returns true when at least one work reached a delivery/stored outcome.
|
|
319
|
+
*/
|
|
320
|
+
async tryInventoryFallback(target, topic, workType) {
|
|
321
|
+
const policy = (0, inventory_1.inventoryPolicy)(target);
|
|
322
|
+
if (!policy.enabled || policy.fallback === false)
|
|
323
|
+
return false;
|
|
324
|
+
const targetId = (0, inventory_1.inventoryTargetId)(target);
|
|
325
|
+
const repo = this.database.candidateInventory;
|
|
326
|
+
while (true) {
|
|
327
|
+
const row = repo.claimNext({ topic, targetId, reserveSize: policy.reserveSize, date: (0, pixiv_date_utils_1.getTodayDate)() });
|
|
328
|
+
if (!row)
|
|
329
|
+
break;
|
|
330
|
+
const id = Number(row.pixivId);
|
|
331
|
+
if (!Number.isFinite(id)) {
|
|
332
|
+
repo.markFiltered(row.pixivId, row.workType, topic, targetId);
|
|
333
|
+
continue;
|
|
334
|
+
}
|
|
335
|
+
let detail;
|
|
336
|
+
try {
|
|
337
|
+
detail = await this.client.getIllustration(id);
|
|
338
|
+
}
|
|
339
|
+
catch (error) {
|
|
340
|
+
this.logError(error, `Inventory candidate illustration ${id} could not be fetched`);
|
|
341
|
+
repo.markSelectedBackToPending(row.pixivId, row.workType, topic, targetId);
|
|
342
|
+
continue;
|
|
343
|
+
}
|
|
344
|
+
const attemptTarget = { ...target };
|
|
345
|
+
const result = await this.pipeline.run([detail], attemptTarget, 'illustration', (illust, tag) => this.downloadAndDeliver(illust, tag, attemptTarget));
|
|
346
|
+
this.scan = (0, TargetOutcome_1.mergeScanSummaries)(this.scan, result.scan);
|
|
347
|
+
if (result.downloaded > 0) {
|
|
348
|
+
this.handleDownloadResult(result, attemptTarget, 'topic', 1);
|
|
349
|
+
(0, inventory_1.markInventoryAttempt)(this.database, target, topic, row.pixivId, 'illustration', 'submitted');
|
|
350
|
+
return true;
|
|
351
|
+
}
|
|
352
|
+
if (this.scan && this.scan.outages.length > 0) {
|
|
353
|
+
// A dead token / dead network is not "no candidate": retry the same
|
|
354
|
+
// row on the next run instead of discarding it.
|
|
355
|
+
(0, inventory_1.markInventoryAttempt)(this.database, target, topic, row.pixivId, 'illustration', 'pending');
|
|
356
|
+
return false;
|
|
357
|
+
}
|
|
358
|
+
// The claimed work is no longer usable (deleted/private/duplicate): drop
|
|
359
|
+
// it permanently and try the next pending row.
|
|
360
|
+
(0, inventory_1.markInventoryAttempt)(this.database, target, topic, row.pixivId, 'illustration', 'filtered');
|
|
361
|
+
}
|
|
362
|
+
return false;
|
|
363
|
+
}
|
|
364
|
+
/** Mark inventory rows whose work reached a delivery/stored outcome. */
|
|
365
|
+
markInventorySubmittedFromOutcomes(target, topic, workType) {
|
|
366
|
+
if (!(0, inventory_1.inventoryPolicy)(target).enabled)
|
|
367
|
+
return;
|
|
368
|
+
for (const outcome of this.outcomes) {
|
|
369
|
+
if (outcome.kind === 'delivery_pending' || outcome.kind === 'submitted' || outcome.kind === 'stored') {
|
|
370
|
+
(0, inventory_1.markInventoryAttempt)(this.database, target, topic, outcome.workId, workType, 'submitted');
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
}
|
|
300
374
|
resolveTopicDay(target) {
|
|
301
375
|
return target.date === 'TODAY'
|
|
302
376
|
? (0, pixiv_date_utils_1.getTodayDate)()
|
|
@@ -67,6 +67,13 @@ export declare class NovelTargetHandler {
|
|
|
67
67
|
private fetchNovels;
|
|
68
68
|
private fetchTopicNovels;
|
|
69
69
|
private handleTopicWithLookback;
|
|
70
|
+
/**
|
|
71
|
+
* Phase 5 fallback: claim pending novel candidates and publish them into the
|
|
72
|
+
* existing aggregate day-scan (same idempotent pipeline as a normal day).
|
|
73
|
+
*/
|
|
74
|
+
private tryInventoryFallback;
|
|
75
|
+
/** Mark inventory rows whose work reached a delivery/stored outcome. */
|
|
76
|
+
private markInventorySubmittedFromOutcomes;
|
|
70
77
|
private resolveTopicDay;
|
|
71
78
|
private shiftDay;
|
|
72
79
|
private fetchRankingNovels;
|
|
@@ -12,6 +12,7 @@ const observability_1 = require("../../observability");
|
|
|
12
12
|
const context_1 = require("../../observability/context");
|
|
13
13
|
const DownloadPlanner_1 = require("../plan/DownloadPlanner");
|
|
14
14
|
const deliveryContext_1 = require("./deliveryContext");
|
|
15
|
+
const inventory_1 = require("../inventory");
|
|
15
16
|
const TelePressRichNovel_1 = require("../../delivery/TelePressRichNovel");
|
|
16
17
|
class NovelTargetHandler {
|
|
17
18
|
client;
|
|
@@ -232,6 +233,7 @@ class NovelTargetHandler {
|
|
|
232
233
|
const pipeline = this.topicPipelineFactory();
|
|
233
234
|
const { works, selection } = await pipeline.selectWorks(target, 'novel', day, selectionLimit, target.topicDiscovery ?? {}, target.candidateCollection ?? {});
|
|
234
235
|
logger_1.logger.info(`Topic "${topic}" novel: tags=${selection.resolvedTagCount} raw=${selection.rawCount} deduped=${selection.dedupedCount} accepted=${selection.acceptedCount} candidates=${works.length} target=${limit}`);
|
|
236
|
+
(0, inventory_1.recordInventoryCandidates)(this.database, target, topic, day, works, 'novel');
|
|
235
237
|
const report = (0, TargetOutcome_1.emptyCandidateSupplyReport)();
|
|
236
238
|
report.fetched = selection.rawCount;
|
|
237
239
|
report.selected = selection.acceptedCount;
|
|
@@ -281,15 +283,84 @@ class NovelTargetHandler {
|
|
|
281
283
|
aggregate.skipped += result.skipped;
|
|
282
284
|
aggregate.alreadyDownloaded += result.alreadyDownloaded;
|
|
283
285
|
aggregate.filteredOut += result.filteredOut;
|
|
286
|
+
if (result.downloaded > 0) {
|
|
287
|
+
this.markInventorySubmittedFromOutcomes(target, (0, inventory_1.inventoryTopic)(target), 'novel');
|
|
288
|
+
}
|
|
284
289
|
if (aggregate.scan.outages.length > 0) {
|
|
285
290
|
// A dead token / dead database / dead network is not "no matching
|
|
286
291
|
// novel": stop looking back and let the job fail/retry.
|
|
287
292
|
break;
|
|
288
293
|
}
|
|
289
294
|
}
|
|
295
|
+
const topic = (0, inventory_1.inventoryTopic)(target);
|
|
296
|
+
await this.tryInventoryFallback(target, topic, aggregate);
|
|
297
|
+
if (this.supplyRep) {
|
|
298
|
+
this.supplyRep = (0, inventory_1.attachInventoryReport)(this.database, target, topic, this.supplyRep);
|
|
299
|
+
}
|
|
290
300
|
this.scan = aggregate.scan;
|
|
291
301
|
await this.handleDownloadResult(aggregate, target, 'topic', totalFound, checkedDays);
|
|
292
302
|
}
|
|
303
|
+
/**
|
|
304
|
+
* Phase 5 fallback: claim pending novel candidates and publish them into the
|
|
305
|
+
* existing aggregate day-scan (same idempotent pipeline as a normal day).
|
|
306
|
+
*/
|
|
307
|
+
async tryInventoryFallback(target, topic, aggregate) {
|
|
308
|
+
const policy = (0, inventory_1.inventoryPolicy)(target);
|
|
309
|
+
if (!policy.enabled || policy.fallback === false)
|
|
310
|
+
return;
|
|
311
|
+
// Only a fully-empty fresh/lookback window may draw from the reserve; a
|
|
312
|
+
// partial day stays on fresh supply rather than padding beyond the limit.
|
|
313
|
+
if (aggregate.downloaded > 0)
|
|
314
|
+
return;
|
|
315
|
+
const targetId = (0, inventory_1.inventoryTargetId)(target);
|
|
316
|
+
const repo = this.database.candidateInventory;
|
|
317
|
+
while (aggregate.downloaded < (target.limit || 1)) {
|
|
318
|
+
const row = repo.claimNext({ topic, targetId, reserveSize: policy.reserveSize, date: (0, pixiv_date_utils_1.getTodayDate)() });
|
|
319
|
+
if (!row)
|
|
320
|
+
break;
|
|
321
|
+
const id = Number(row.pixivId);
|
|
322
|
+
if (!Number.isFinite(id)) {
|
|
323
|
+
repo.markFiltered(row.pixivId, row.workType, topic, targetId);
|
|
324
|
+
continue;
|
|
325
|
+
}
|
|
326
|
+
let detail;
|
|
327
|
+
try {
|
|
328
|
+
detail = await this.client.getNovelDetail(id);
|
|
329
|
+
}
|
|
330
|
+
catch (error) {
|
|
331
|
+
this.logError(error, `Inventory candidate novel ${id} could not be fetched`);
|
|
332
|
+
repo.markSelectedBackToPending(row.pixivId, row.workType, topic, targetId);
|
|
333
|
+
continue;
|
|
334
|
+
}
|
|
335
|
+
const attemptTarget = {
|
|
336
|
+
...target,
|
|
337
|
+
limit: (target.limit || 1) - aggregate.downloaded,
|
|
338
|
+
};
|
|
339
|
+
const result = await this.pipeline.run([detail], attemptTarget, 'novel', (novel, tag) => this.downloadAndDeliver(novel, tag, attemptTarget));
|
|
340
|
+
aggregate.scan = (0, TargetOutcome_1.mergeScanSummaries)(aggregate.scan, result.scan);
|
|
341
|
+
aggregate.downloaded += result.downloaded;
|
|
342
|
+
aggregate.skipped += result.skipped;
|
|
343
|
+
aggregate.alreadyDownloaded += result.alreadyDownloaded;
|
|
344
|
+
aggregate.filteredOut += result.filteredOut;
|
|
345
|
+
if (result.downloaded > 0) {
|
|
346
|
+
(0, inventory_1.markInventoryAttempt)(this.database, target, topic, row.pixivId, 'novel', 'submitted');
|
|
347
|
+
continue;
|
|
348
|
+
}
|
|
349
|
+
if (aggregate.scan.outages.length > 0)
|
|
350
|
+
return;
|
|
351
|
+
(0, inventory_1.markInventoryAttempt)(this.database, target, topic, row.pixivId, 'novel', 'filtered');
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
/** Mark inventory rows whose work reached a delivery/stored outcome. */
|
|
355
|
+
markInventorySubmittedFromOutcomes(target, topic, workType) {
|
|
356
|
+
if (!(0, inventory_1.inventoryPolicy)(target).enabled)
|
|
357
|
+
return;
|
|
358
|
+
for (const outcome of this.outcomes) {
|
|
359
|
+
if (outcome.kind === 'delivery_pending' || outcome.kind === 'submitted' || outcome.kind === 'stored') {
|
|
360
|
+
(0, inventory_1.markInventoryAttempt)(this.database, target, topic, outcome.workId, workType, 'submitted');
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
}
|
|
293
364
|
resolveTopicDay(target) {
|
|
294
365
|
return target.date === 'TODAY'
|
|
295
366
|
? (0, pixiv_date_utils_1.getTodayDate)()
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { TargetConfig } from '../config';
|
|
2
|
+
import type { IDatabase } from '../interfaces/IDatabase';
|
|
3
|
+
import type { CandidateSupplyReport } from '../scheduler/TargetOutcome';
|
|
4
|
+
/** Defaults and bounds for the optional Phase 5 inventory. */
|
|
5
|
+
export interface InventoryPolicy {
|
|
6
|
+
enabled: boolean;
|
|
7
|
+
maxAgeDays: number;
|
|
8
|
+
reserveSize: number;
|
|
9
|
+
fallback: boolean;
|
|
10
|
+
}
|
|
11
|
+
export declare function inventoryPolicy(target: TargetConfig): InventoryPolicy;
|
|
12
|
+
export declare function inventoryTopic(target: TargetConfig): string;
|
|
13
|
+
export declare function inventoryTargetId(target: TargetConfig): string;
|
|
14
|
+
/**
|
|
15
|
+
* Phase 5: after a topic scan yields eligible works that were NOT delivered by
|
|
16
|
+
* this run's lookback loop, upsert them as idle pending inventory. Works that
|
|
17
|
+
* are already recorded as downloaded/delivered are skipped by the caller
|
|
18
|
+
* upstream (selectWorks already removed download-history rows); this method is
|
|
19
|
+
* deliberately idempotent.
|
|
20
|
+
*/
|
|
21
|
+
export declare function recordInventoryCandidates(database: IDatabase, target: TargetConfig, topic: string, date: string, works: Array<{
|
|
22
|
+
id: number;
|
|
23
|
+
type?: string;
|
|
24
|
+
}>, workType: 'illustration' | 'novel'): void;
|
|
25
|
+
/** Enrich a candidate report with the durable pending-count snapshot. */
|
|
26
|
+
export declare function attachInventoryReport(database: IDatabase, target: TargetConfig, topic: string, report: CandidateSupplyReport): CandidateSupplyReport;
|
|
27
|
+
/** Marks a claimed inventory row after a delivery attempt. */
|
|
28
|
+
export declare function markInventoryAttempt(database: IDatabase, target: TargetConfig, topic: string, workId: string, workType: 'illustration' | 'novel', outcome: 'submitted' | 'filtered' | 'pending'): void;
|
|
29
|
+
//# sourceMappingURL=inventory.d.ts.map
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.inventoryPolicy = inventoryPolicy;
|
|
4
|
+
exports.inventoryTopic = inventoryTopic;
|
|
5
|
+
exports.inventoryTargetId = inventoryTargetId;
|
|
6
|
+
exports.recordInventoryCandidates = recordInventoryCandidates;
|
|
7
|
+
exports.attachInventoryReport = attachInventoryReport;
|
|
8
|
+
exports.markInventoryAttempt = markInventoryAttempt;
|
|
9
|
+
function inventoryPolicy(target) {
|
|
10
|
+
const cfg = target.topicProfile?.inventory;
|
|
11
|
+
return {
|
|
12
|
+
enabled: cfg?.enabled === true,
|
|
13
|
+
maxAgeDays: cfg?.maxAgeDays ?? 30,
|
|
14
|
+
reserveSize: Math.min(Math.max(cfg?.reserveSize ?? 20, 1), 100),
|
|
15
|
+
fallback: cfg?.fallback ?? true,
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
function inventoryTopic(target) {
|
|
19
|
+
return (target.topicProfile?.primary?.[0] ?? target.topic ?? '').trim();
|
|
20
|
+
}
|
|
21
|
+
function inventoryTargetId(target) {
|
|
22
|
+
return target.id ?? inventoryTopic(target);
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Phase 5: after a topic scan yields eligible works that were NOT delivered by
|
|
26
|
+
* this run's lookback loop, upsert them as idle pending inventory. Works that
|
|
27
|
+
* are already recorded as downloaded/delivered are skipped by the caller
|
|
28
|
+
* upstream (selectWorks already removed download-history rows); this method is
|
|
29
|
+
* deliberately idempotent.
|
|
30
|
+
*/
|
|
31
|
+
function recordInventoryCandidates(database, target, topic, date, works, workType) {
|
|
32
|
+
const policy = inventoryPolicy(target);
|
|
33
|
+
if (!policy.enabled)
|
|
34
|
+
return;
|
|
35
|
+
const targetId = inventoryTargetId(target);
|
|
36
|
+
for (const work of works) {
|
|
37
|
+
const id = String(work?.id);
|
|
38
|
+
if (!id)
|
|
39
|
+
continue;
|
|
40
|
+
database.candidateInventory.upsert({
|
|
41
|
+
pixivId: id,
|
|
42
|
+
workType,
|
|
43
|
+
topic,
|
|
44
|
+
targetId,
|
|
45
|
+
snapshot: work,
|
|
46
|
+
date,
|
|
47
|
+
maxAgeDays: policy.maxAgeDays,
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
/** Enrich a candidate report with the durable pending-count snapshot. */
|
|
52
|
+
function attachInventoryReport(database, target, topic, report) {
|
|
53
|
+
const policy = inventoryPolicy(target);
|
|
54
|
+
if (!policy.enabled)
|
|
55
|
+
return report;
|
|
56
|
+
const summary = database.candidateInventory.pendingSummary(topic, inventoryTargetId(target));
|
|
57
|
+
return {
|
|
58
|
+
...report,
|
|
59
|
+
inventory: {
|
|
60
|
+
pendingCount: summary.count,
|
|
61
|
+
reserveSize: policy.reserveSize,
|
|
62
|
+
maxAgeDays: policy.maxAgeDays,
|
|
63
|
+
oldestSeenDate: summary.oldestSeenDate,
|
|
64
|
+
},
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
/** Marks a claimed inventory row after a delivery attempt. */
|
|
68
|
+
function markInventoryAttempt(database, target, topic, workId, workType, outcome) {
|
|
69
|
+
if (!inventoryPolicy(target).enabled)
|
|
70
|
+
return;
|
|
71
|
+
const targetId = inventoryTargetId(target);
|
|
72
|
+
const repo = database.candidateInventory;
|
|
73
|
+
if (outcome === 'submitted')
|
|
74
|
+
repo.markSubmitted(workId, workType, topic, targetId);
|
|
75
|
+
else if (outcome === 'filtered')
|
|
76
|
+
repo.markFiltered(workId, workType, topic, targetId);
|
|
77
|
+
else
|
|
78
|
+
repo.markSelectedBackToPending(workId, workType, topic, targetId);
|
|
79
|
+
}
|
|
80
|
+
//# sourceMappingURL=inventory.js.map
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { AccessTokenStore, DownloadRecordInput, ExecutionStatus, SchedulerExecutionRecord } from '../storage/Database';
|
|
2
|
+
import { CandidateInventoryRepository } from '../storage/repositories/CandidateInventoryRepository';
|
|
2
3
|
/**
|
|
3
4
|
* Interface for database operations
|
|
4
5
|
* Provides abstraction for data persistence
|
|
@@ -123,5 +124,7 @@ export interface IDatabase {
|
|
|
123
124
|
};
|
|
124
125
|
countSince(botId: string | null, hours: number): number;
|
|
125
126
|
};
|
|
127
|
+
/** Phase 5 durable CandidateInventory (待发池). */
|
|
128
|
+
readonly candidateInventory: CandidateInventoryRepository;
|
|
126
129
|
}
|
|
127
130
|
//# sourceMappingURL=IDatabase.d.ts.map
|
package/dist/package.json
CHANGED
|
@@ -109,6 +109,17 @@ export interface CandidateSupplyReason {
|
|
|
109
109
|
code: string;
|
|
110
110
|
count: number;
|
|
111
111
|
}
|
|
112
|
+
/** Phase 5: durable 待发池 snapshot for sparse topics (only when enabled). */
|
|
113
|
+
export interface CandidateInventoryReport {
|
|
114
|
+
/** Rows still usable for fallback at report time. */
|
|
115
|
+
pendingCount: number;
|
|
116
|
+
/** Target's configured max retained rows. */
|
|
117
|
+
reserveSize: number;
|
|
118
|
+
/** Max age in days before a pending candidate expires. */
|
|
119
|
+
maxAgeDays: number;
|
|
120
|
+
/** Earliest still-pending first-seen date (YYYY-MM-DD), null when empty. */
|
|
121
|
+
oldestSeenDate: string | null;
|
|
122
|
+
}
|
|
112
123
|
/** Candidate-supply observability snapshot (Phase 1 Candidate Report). */
|
|
113
124
|
export interface CandidateSupplyReport {
|
|
114
125
|
/** Total works surfaced by the topic search before any filtering. */
|
|
@@ -119,6 +130,8 @@ export interface CandidateSupplyReport {
|
|
|
119
130
|
rejected: number;
|
|
120
131
|
/** Why candidates were rejected, by reason code (extensible). */
|
|
121
132
|
reasons: CandidateSupplyReason[];
|
|
133
|
+
/** Phase 5: durable 待发池 reserve (optional, present only when enabled). */
|
|
134
|
+
inventory?: CandidateInventoryReport;
|
|
122
135
|
}
|
|
123
136
|
/**
|
|
124
137
|
* A freshly-harvested upstream funnel with no candidates selected yet.
|
|
@@ -20,7 +20,7 @@ exports.terminalReasonFor = terminalReasonFor;
|
|
|
20
20
|
* A freshly-harvested upstream funnel with no candidates selected yet.
|
|
21
21
|
*/
|
|
22
22
|
function emptyCandidateSupplyReport() {
|
|
23
|
-
return { fetched: 0, selected: 0, rejected: 0, reasons: [] };
|
|
23
|
+
return { fetched: 0, selected: 0, rejected: 0, reasons: [], inventory: undefined };
|
|
24
24
|
}
|
|
25
25
|
/**
|
|
26
26
|
* Fold two supply snapshots of the SAME logical target together, so a
|
|
@@ -43,6 +43,7 @@ function mergeCandidateSupplyReports(first, second) {
|
|
|
43
43
|
selected: first.selected + second.selected,
|
|
44
44
|
rejected: Math.max(0, first.fetched + second.fetched - (first.selected + second.selected)),
|
|
45
45
|
reasons,
|
|
46
|
+
inventory: second.inventory ?? first.inventory,
|
|
46
47
|
};
|
|
47
48
|
}
|
|
48
49
|
/**
|
|
@@ -68,6 +69,7 @@ function withScanSkips(report, scan) {
|
|
|
68
69
|
selected,
|
|
69
70
|
rejected: Math.max(0, base.fetched - selected),
|
|
70
71
|
reasons,
|
|
72
|
+
inventory: base.inventory,
|
|
71
73
|
};
|
|
72
74
|
}
|
|
73
75
|
/**
|
|
@@ -5,6 +5,7 @@ import { OutboxRepository } from './repositories/OutboxRepository';
|
|
|
5
5
|
import { MetadataRepository } from './repositories/MetadataRepository';
|
|
6
6
|
import { SQLiteRateLimitStateStore } from './repositories/RateLimitStateRepository';
|
|
7
7
|
import { SystemErrorRepository } from './repositories/SystemErrorRepository';
|
|
8
|
+
import { CandidateInventoryRepository } from './repositories/CandidateInventoryRepository';
|
|
8
9
|
export interface AccessTokenStore {
|
|
9
10
|
accessToken: string;
|
|
10
11
|
expiresAt: number;
|
|
@@ -48,6 +49,7 @@ export declare class Database implements IDatabase {
|
|
|
48
49
|
private metadataRepo;
|
|
49
50
|
private rateLimitStateStore;
|
|
50
51
|
private systemErrorRepo;
|
|
52
|
+
private candidateInventoryRepo;
|
|
51
53
|
constructor(databasePath: string);
|
|
52
54
|
migrate(): void;
|
|
53
55
|
/** Absolute path of the SQLite file (used to locate sibling cache dirs). */
|
|
@@ -64,6 +66,8 @@ export declare class Database implements IDatabase {
|
|
|
64
66
|
get rateLimitState(): SQLiteRateLimitStateStore;
|
|
65
67
|
/** Durable system-error ledger (observability; never throws). */
|
|
66
68
|
get systemErrors(): SystemErrorRepository;
|
|
69
|
+
/** Phase 5 CandidateInventory (待发池) for sparse topics. */
|
|
70
|
+
get candidateInventory(): CandidateInventoryRepository;
|
|
67
71
|
/** Raw transactional boundary for atomic multi-table intents. */
|
|
68
72
|
transaction<T>(fn: () => T): T;
|
|
69
73
|
/** Expose a prepared-statement helper if needed by services (pragmas etc). */
|
package/dist/storage/Database.js
CHANGED
|
@@ -18,6 +18,7 @@ const OutboxRepository_1 = require("./repositories/OutboxRepository");
|
|
|
18
18
|
const MetadataRepository_1 = require("./repositories/MetadataRepository");
|
|
19
19
|
const RateLimitStateRepository_1 = require("./repositories/RateLimitStateRepository");
|
|
20
20
|
const SystemErrorRepository_1 = require("./repositories/SystemErrorRepository");
|
|
21
|
+
const CandidateInventoryRepository_1 = require("./repositories/CandidateInventoryRepository");
|
|
21
22
|
const NodeSqliteDriver_1 = require("./drivers/NodeSqliteDriver");
|
|
22
23
|
class Database {
|
|
23
24
|
databasePath;
|
|
@@ -35,6 +36,7 @@ class Database {
|
|
|
35
36
|
metadataRepo;
|
|
36
37
|
rateLimitStateStore;
|
|
37
38
|
systemErrorRepo;
|
|
39
|
+
candidateInventoryRepo;
|
|
38
40
|
constructor(databasePath) {
|
|
39
41
|
this.databasePath = databasePath;
|
|
40
42
|
try {
|
|
@@ -63,6 +65,7 @@ class Database {
|
|
|
63
65
|
this.metadataRepo = new MetadataRepository_1.MetadataRepository(this.db);
|
|
64
66
|
this.rateLimitStateStore = new RateLimitStateRepository_1.SQLiteRateLimitStateStore(this.db);
|
|
65
67
|
this.systemErrorRepo = new SystemErrorRepository_1.SystemErrorRepository(this.db);
|
|
68
|
+
this.candidateInventoryRepo = new CandidateInventoryRepository_1.CandidateInventoryRepository(this.db);
|
|
66
69
|
}
|
|
67
70
|
catch (error) {
|
|
68
71
|
throw new errors_1.DatabaseError(`Failed to initialize database at ${this.databasePath}`, error instanceof Error ? error : undefined);
|
|
@@ -102,6 +105,10 @@ class Database {
|
|
|
102
105
|
get systemErrors() {
|
|
103
106
|
return this.systemErrorRepo;
|
|
104
107
|
}
|
|
108
|
+
/** Phase 5 CandidateInventory (待发池) for sparse topics. */
|
|
109
|
+
get candidateInventory() {
|
|
110
|
+
return this.candidateInventoryRepo;
|
|
111
|
+
}
|
|
105
112
|
/** Raw transactional boundary for atomic multi-table intents. */
|
|
106
113
|
transaction(fn) {
|
|
107
114
|
return this.db.transaction(fn)();
|