pixivflow 3.0.2 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/TopicCommand.js +41 -9
- package/dist/config/types.d.ts +65 -1
- package/dist/config/types.js +3 -0
- package/dist/config/validation.js +47 -0
- package/dist/domain/media/NovelCoverPolicy.d.ts +28 -0
- package/dist/domain/media/NovelCoverPolicy.js +70 -0
- package/dist/download/DownloadManager.js +5 -1
- package/dist/download/NovelDownloader.d.ts +23 -1
- package/dist/download/NovelDownloader.js +69 -5
- package/dist/download/novelCover.d.ts +29 -5
- package/dist/download/novelCover.js +34 -5
- package/dist/package.json +1 -1
- package/dist/topic/TopicPipeline.d.ts +55 -1
- package/dist/topic/TopicPipeline.js +256 -41
- package/dist/topic/TopicResolver.js +4 -1
- package/dist/topic/TopicTagScorer.js +12 -1
- package/dist/topic/types.d.ts +62 -0
- package/dist/utils/imageDimensions.d.ts +19 -0
- package/dist/utils/imageDimensions.js +95 -0
- package/dist/version.js +1 -1
- package/dist/webui/package.json +1 -1
- package/package.json +1 -1
|
@@ -7,6 +7,7 @@ const Database_1 = require("../storage/Database");
|
|
|
7
7
|
const PixivAuth_1 = require("../auth/PixivAuth");
|
|
8
8
|
const createPixivFlowClient_1 = require("../pixiv-client/createPixivFlowClient");
|
|
9
9
|
const createTopicPipeline_1 = require("../topic/createTopicPipeline");
|
|
10
|
+
const TopicPipeline_1 = require("../topic/TopicPipeline");
|
|
10
11
|
const TopicResolver_1 = require("../topic/TopicResolver");
|
|
11
12
|
const TopicCache_1 = require("../topic/TopicCache");
|
|
12
13
|
const node_path_1 = require("node:path");
|
|
@@ -50,27 +51,48 @@ class TopicCommand extends Command_1.BaseCommand {
|
|
|
50
51
|
database.close();
|
|
51
52
|
}
|
|
52
53
|
}
|
|
53
|
-
async resolve(context,
|
|
54
|
+
async resolve(context, args, topic) {
|
|
54
55
|
if (!topic) {
|
|
55
56
|
console.error('\nUsage: pixivflow topic resolve <topic> [--type illustration|novel|all] [--refresh]\n');
|
|
56
57
|
return this.failure('Missing topic for resolve');
|
|
57
58
|
}
|
|
58
|
-
const type = (String(
|
|
59
|
-
const refresh = Boolean(
|
|
59
|
+
const type = (String(args.options.type ?? 'all'));
|
|
60
|
+
const refresh = Boolean(args.options.refresh);
|
|
60
61
|
const types = type === 'all' ? ['illustration', 'novel'] : [type];
|
|
61
62
|
return this.withClient(context, async ({ client, database }) => {
|
|
62
63
|
const cache = new TopicCache_1.TopicCache((0, node_path_1.dirname)(database.getDatabasePath()) + '/topic-cache');
|
|
64
|
+
const discovery = context.config.targets?.find((target) => target.mode === 'topic')?.topicDiscovery ?? {};
|
|
65
|
+
// §tag-provenance: the audit surface for "the original tag dominates and no
|
|
66
|
+
// related tag outranks it". Each row exposes where the tag came from, its
|
|
67
|
+
// semantic weight and whether the current tagRelations/relatedTags settings
|
|
68
|
+
// would actually search it today (the `searched` column).
|
|
69
|
+
const rows = [];
|
|
70
|
+
const lines = [];
|
|
63
71
|
for (const contentType of types) {
|
|
64
72
|
const resolver = new TopicResolver_1.TopicResolver(client, cache, context.config.download?.requestDelay ?? 500);
|
|
65
73
|
const { space, fromCache, degraded } = await resolver.resolve(topic, contentType, { refresh });
|
|
66
|
-
|
|
67
|
-
|
|
74
|
+
const walked = (0, TopicPipeline_1.selectWalkedTags)(space.tags, key(topic), discovery.tagRelations);
|
|
75
|
+
const channels = new Set((0, TopicPipeline_1.recallChannels)(walked, key(topic), relatedMode(discovery.relatedTags)).map((tag) => key(tag.name)));
|
|
76
|
+
lines.push('');
|
|
77
|
+
lines.push(`Topic: ${topic} (${contentType}) ${fromCache ? (degraded ? '· stale cache' : '· cache') : '· fresh'}`);
|
|
78
|
+
lines.push('Name'.padEnd(24) + 'Trans'.padEnd(16) + 'Source'.padEnd(28) + 'Weight'.padStart(7) + 'Score'.padStart(7) + 'Seed'.padStart(6) + 'Searched'.padStart(10));
|
|
79
|
+
const tagRows = [];
|
|
68
80
|
for (const tag of space.tags) {
|
|
69
|
-
const
|
|
70
|
-
|
|
81
|
+
const source = tag.source ?? (tag.seed ? 'seed' : 'cooccurrence');
|
|
82
|
+
const weight = tag.weight ?? tag.score;
|
|
83
|
+
const searched = channels.has(key(tag.name));
|
|
84
|
+
tagRows.push({ name: tag.name, translatedName: tag.translatedName, source, weight, score: tag.score, seed: tag.seed, searched });
|
|
85
|
+
lines.push(tag.name.slice(0, 23).padEnd(24)
|
|
86
|
+
+ (tag.translatedName ?? '-').slice(0, 15).padEnd(16)
|
|
87
|
+
+ source.padEnd(28)
|
|
88
|
+
+ weight.toFixed(2).padStart(7)
|
|
89
|
+
+ tag.score.toFixed(2).padStart(7)
|
|
90
|
+
+ (tag.seed ? 'yes' : 'no').padStart(6)
|
|
91
|
+
+ (searched ? 'yes' : 'no').padStart(10));
|
|
71
92
|
}
|
|
93
|
+
rows.push({ contentType, fromCache, degraded, tags: tagRows });
|
|
72
94
|
}
|
|
73
|
-
return this.success('
|
|
95
|
+
return this.success(lines.join('\n'), { topic, types: rows });
|
|
74
96
|
});
|
|
75
97
|
}
|
|
76
98
|
async test(context, args, topic) {
|
|
@@ -92,6 +114,7 @@ class TopicCommand extends Command_1.BaseCommand {
|
|
|
92
114
|
const { works, selection } = await pipeline.selectWorks(target, contentType, day, limit, { refresh }, {});
|
|
93
115
|
console.log(`\n=== ${contentType} topic "${topic}" day ${day} ===`);
|
|
94
116
|
console.log(`resolvedTags=${selection.resolvedTagCount} raw=${selection.rawCount} deduped=${selection.dedupedCount} accepted=${selection.acceptedCount} selected=${works.length}`);
|
|
117
|
+
console.log(`searchedTags=${(selection.searchedTags ?? []).join(', ')}`);
|
|
95
118
|
selection.selected.forEach((c, i) => {
|
|
96
119
|
console.log(` #${i + 1} id=${c.id} pop=${c.popularity.toFixed(1)} meta=${c.metadataScore.toFixed(2)} ${c.title}`);
|
|
97
120
|
});
|
|
@@ -103,7 +126,8 @@ class TopicCommand extends Command_1.BaseCommand {
|
|
|
103
126
|
getUsage() {
|
|
104
127
|
return [
|
|
105
128
|
'topic resolve <topic> [--type all|illustration|novel] [--refresh]',
|
|
106
|
-
' Show the Pixiv-derived related tag space for a topic (cached)
|
|
129
|
+
' Show the Pixiv-derived related tag space for a topic (cached), with each',
|
|
130
|
+
' tag\'s provenance (source), semantic weight, score and whether it is searched.',
|
|
107
131
|
'',
|
|
108
132
|
'topic test <topic> [--type all|illustration|novel] [--date YESTERDAY|YYYY-MM-DD] [--limit N] [--refresh]',
|
|
109
133
|
' Dry-run a daily selection: resolved tags, candidate counts, Top N (no downloads).',
|
|
@@ -111,4 +135,12 @@ class TopicCommand extends Command_1.BaseCommand {
|
|
|
111
135
|
}
|
|
112
136
|
}
|
|
113
137
|
exports.TopicCommand = TopicCommand;
|
|
138
|
+
/** Same normalization as the pipeline/resolver keys (trim + NFKC + lowercase). */
|
|
139
|
+
function key(value) {
|
|
140
|
+
return value.normalize('NFKC').trim().toLocaleLowerCase();
|
|
141
|
+
}
|
|
142
|
+
/** Unknown modes fall back to the historical 'always', as in the pipeline. */
|
|
143
|
+
function relatedMode(value) {
|
|
144
|
+
return value === 'when_seed_insufficient' || value === 'never' ? value : 'always';
|
|
145
|
+
}
|
|
114
146
|
//# sourceMappingURL=TopicCommand.js.map
|
package/dist/config/types.d.ts
CHANGED
|
@@ -4,6 +4,25 @@
|
|
|
4
4
|
export type TargetType = 'illustration' | 'novel';
|
|
5
5
|
import type { DeliveryCapabilityOverrides } from '../delivery/capabilities';
|
|
6
6
|
export type DeliveryFieldValue = string | number | boolean | string[];
|
|
7
|
+
/** Provenance names accepted by `TopicDiscoveryConfig.tagRelations`. */
|
|
8
|
+
export declare const TAG_RELATION_SOURCES: readonly ["seed", "cooccurrence", "autocomplete"];
|
|
9
|
+
export type TagRelationSource = (typeof TAG_RELATION_SOURCES)[number];
|
|
10
|
+
/**
|
|
11
|
+
* Which resolved tags may become recall channels (§tag-provenance). Every list
|
|
12
|
+
* is optional and the defaults reproduce the pre-existing behaviour: walk the
|
|
13
|
+
* whole resolved space.
|
|
14
|
+
*
|
|
15
|
+
* `deny` always wins, then `allowSources`, then `allow`. The seed tag is never
|
|
16
|
+
* dropped by `allow`/`allowSources` — only `deny` can drop it.
|
|
17
|
+
*/
|
|
18
|
+
export interface TagRelationsConfig {
|
|
19
|
+
/** Provenance categories allowed to be walked (default: all three). */
|
|
20
|
+
allowSources?: TagRelationSource[];
|
|
21
|
+
/** If non-empty: ONLY these tag names are walked (the seed tag is kept). */
|
|
22
|
+
allow?: string[];
|
|
23
|
+
/** Always dropped, wins over `allow`/`allowSources` (default: none). */
|
|
24
|
+
deny?: string[];
|
|
25
|
+
}
|
|
7
26
|
/** Discovery tuning for mode='topic'. Every value has a safe default. */
|
|
8
27
|
export interface TopicDiscoveryConfig {
|
|
9
28
|
/** Max related tags used to build the search space (default 12). */
|
|
@@ -12,12 +31,42 @@ export interface TopicDiscoveryConfig {
|
|
|
12
31
|
sampleWorks?: number;
|
|
13
32
|
/** Cache lifetime in days for the resolved tag space (default 7). */
|
|
14
33
|
cacheDays?: number;
|
|
15
|
-
/** Minimum relatedness score for a tag to enter the space (default 0.
|
|
34
|
+
/** Minimum relatedness score for a tag to enter the space (default 0.22). */
|
|
16
35
|
minScore?: number;
|
|
17
36
|
/** Ignore a fresh cache and re-discover now (default false). */
|
|
18
37
|
refresh?: boolean;
|
|
19
38
|
/** Include R-18 works in topic sampling and collection (default false). */
|
|
20
39
|
includeR18?: boolean;
|
|
40
|
+
/**
|
|
41
|
+
* When related tags may be searched as their own recall channel (§topic-recall).
|
|
42
|
+
*
|
|
43
|
+
* - `'always'` (default): walk the whole resolved tag space every day.
|
|
44
|
+
* - `'when_seed_insufficient'`: search the topic tag first and only fall back
|
|
45
|
+
* to related tags when it cannot fill `limit` for that day — a related tag
|
|
46
|
+
* is a hint, never a substitute for the topic the operator asked for.
|
|
47
|
+
* - `'never'`: search the topic tag alone.
|
|
48
|
+
*/
|
|
49
|
+
relatedTags?: 'always' | 'when_seed_insufficient' | 'never';
|
|
50
|
+
/**
|
|
51
|
+
* Make the seed tag a HARD ranking tier (default `'off'`).
|
|
52
|
+
*
|
|
53
|
+
* `'off'` keeps the documented popularity-only ranking, where relevance is
|
|
54
|
+
* only an acceptance gate. `'on'` guarantees the invariant "原始 Tag 权重最高,
|
|
55
|
+
* 相关 Tag 权重不得超过原始 Tag": a work carrying the seed tag always outranks
|
|
56
|
+
* a work that only matched expanded tags, however popular the latter is.
|
|
57
|
+
*/
|
|
58
|
+
seedTier?: 'off' | 'on';
|
|
59
|
+
/** Which resolved tags may be walked as recall channels (default: all). */
|
|
60
|
+
tagRelations?: TagRelationsConfig;
|
|
61
|
+
/**
|
|
62
|
+
* Also treat a work's `translated_name` as a tag hit (default false).
|
|
63
|
+
*
|
|
64
|
+
* Today only the work's `name` is compared with the resolved tag keys. When
|
|
65
|
+
* enabled, a work whose translated tag name matches a resolved tag counts as
|
|
66
|
+
* carrying that tag (and a translated match of the seed counts as a seed hit).
|
|
67
|
+
* Default `false` keeps today's matching exactly.
|
|
68
|
+
*/
|
|
69
|
+
matchTranslatedNames?: boolean;
|
|
21
70
|
}
|
|
22
71
|
/** Candidate collection tuning for mode='topic'. */
|
|
23
72
|
export interface CandidateCollectionConfig {
|
|
@@ -866,6 +915,21 @@ export interface StandaloneConfig {
|
|
|
866
915
|
* Default: 'eager'
|
|
867
916
|
*/
|
|
868
917
|
materializationPolicy?: 'eager' | 'on-demand';
|
|
918
|
+
/**
|
|
919
|
+
* Novel cover content policy (§media-asset-pipeline / §novel-cover).
|
|
920
|
+
*
|
|
921
|
+
* Pixiv serves author covers and its own generated design covers from the
|
|
922
|
+
* same URL shape, so the content type is classified from the fetched bytes:
|
|
923
|
+
* a generated design (exactly 640x900) is never delivered as Telegram media.
|
|
924
|
+
* This key governs the UNCERTAIN case only — a cover whose header cannot be
|
|
925
|
+
* classified. 'skip' (default) is safe mode: never ship an unclassifiable
|
|
926
|
+
* cover, and log `coverType=unknown` so a future Pixiv format change is
|
|
927
|
+
* visible instead of silently leaking designs again. 'keep' prefers
|
|
928
|
+
* availability over certainty. A failed probe always keeps the cover.
|
|
929
|
+
*/
|
|
930
|
+
novelCover?: {
|
|
931
|
+
unknown?: 'skip' | 'keep';
|
|
932
|
+
};
|
|
869
933
|
};
|
|
870
934
|
}
|
|
871
935
|
//# sourceMappingURL=types.d.ts.map
|
package/dist/config/types.js
CHANGED
|
@@ -3,4 +3,7 @@
|
|
|
3
3
|
* Configuration type definitions
|
|
4
4
|
*/
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.TAG_RELATION_SOURCES = void 0;
|
|
7
|
+
/** Provenance names accepted by `TopicDiscoveryConfig.tagRelations`. */
|
|
8
|
+
exports.TAG_RELATION_SOURCES = ['seed', 'cooccurrence', 'autocomplete'];
|
|
6
9
|
//# sourceMappingURL=types.js.map
|
|
@@ -191,6 +191,48 @@ function validateConfig(config, location, databasePath) {
|
|
|
191
191
|
if (td.includeR18 !== undefined && typeof td.includeR18 !== 'boolean') {
|
|
192
192
|
errors.push(`targets[${index}].topicDiscovery.includeR18: Must be a boolean (got ${typeof td.includeR18})`);
|
|
193
193
|
}
|
|
194
|
+
if (td.relatedTags !== undefined && !['always', 'when_seed_insufficient', 'never'].includes(td.relatedTags)) {
|
|
195
|
+
errors.push(`targets[${index}].topicDiscovery.relatedTags: Must be "always", "when_seed_insufficient" or "never" (got ${String(td.relatedTags)})`);
|
|
196
|
+
}
|
|
197
|
+
// §tag-provenance: all optional; the defaults reproduce the pre-change
|
|
198
|
+
// behaviour (whole space walked, popularity-only ranking, name-only match).
|
|
199
|
+
if (td.seedTier !== undefined && !['off', 'on'].includes(td.seedTier)) {
|
|
200
|
+
errors.push(`targets[${index}].topicDiscovery.seedTier: Must be "off" or "on" (got ${String(td.seedTier)})`);
|
|
201
|
+
}
|
|
202
|
+
if (td.matchTranslatedNames !== undefined && typeof td.matchTranslatedNames !== 'boolean') {
|
|
203
|
+
errors.push(`targets[${index}].topicDiscovery.matchTranslatedNames: Must be a boolean (got ${typeof td.matchTranslatedNames})`);
|
|
204
|
+
}
|
|
205
|
+
const relations = td.tagRelations;
|
|
206
|
+
if (relations !== undefined) {
|
|
207
|
+
if (typeof relations !== 'object' || relations === null || Array.isArray(relations)) {
|
|
208
|
+
errors.push(`targets[${index}].topicDiscovery.tagRelations: Must be an object (got ${Array.isArray(relations) ? 'array' : typeof relations})`);
|
|
209
|
+
}
|
|
210
|
+
else {
|
|
211
|
+
const knownSources = ['seed', 'cooccurrence', 'autocomplete'];
|
|
212
|
+
const listErrors = (field) => (field === 'allowSources'
|
|
213
|
+
? `targets[${index}].topicDiscovery.tagRelations.allowSources: Must be an array of tag sources: ${knownSources.map((s) => `"${s}"`).join(', ')} (got ${JSON.stringify(relations.allowSources)})`
|
|
214
|
+
: `targets[${index}].topicDiscovery.tagRelations.${field}: Must be an array of tag names (got ${JSON.stringify(relations[field])})`);
|
|
215
|
+
const checkList = (field) => {
|
|
216
|
+
const value = relations[field];
|
|
217
|
+
if (value === undefined)
|
|
218
|
+
return undefined;
|
|
219
|
+
if (!Array.isArray(value) || value.some((entry) => typeof entry !== 'string' || entry.trim() === '')) {
|
|
220
|
+
errors.push(listErrors(field));
|
|
221
|
+
return undefined;
|
|
222
|
+
}
|
|
223
|
+
return value;
|
|
224
|
+
};
|
|
225
|
+
const allowSources = checkList('allowSources');
|
|
226
|
+
if (allowSources) {
|
|
227
|
+
const unknown = allowSources.filter((source) => !knownSources.includes(source));
|
|
228
|
+
if (unknown.length > 0) {
|
|
229
|
+
errors.push(`targets[${index}].topicDiscovery.tagRelations.allowSources: Unknown tag source${unknown.length > 1 ? 's' : ''} ${unknown.map((source) => `"${source}"`).join(', ')}; known sources are ${knownSources.map((s) => `"${s}"`).join(', ')}`);
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
checkList('allow');
|
|
233
|
+
checkList('deny');
|
|
234
|
+
}
|
|
235
|
+
}
|
|
194
236
|
}
|
|
195
237
|
const cc = target.candidateCollection;
|
|
196
238
|
if (cc) {
|
|
@@ -657,6 +699,11 @@ function validateConfig(config, location, databasePath) {
|
|
|
657
699
|
config.download.candidateScanLimit > 100)) {
|
|
658
700
|
warnings.push('download.candidateScanLimit: Should be an integer between 1 and 100');
|
|
659
701
|
}
|
|
702
|
+
if (config.download.novelCover?.unknown !== undefined &&
|
|
703
|
+
config.download.novelCover.unknown !== 'skip' &&
|
|
704
|
+
config.download.novelCover.unknown !== 'keep') {
|
|
705
|
+
errors.push(`download.novelCover.unknown: Must be "skip" or "keep" (got ${String(config.download.novelCover.unknown)})`);
|
|
706
|
+
}
|
|
660
707
|
}
|
|
661
708
|
// The bounded candidate scan is what stops a page full of duplicates from
|
|
662
709
|
// burning a whole scheduled slot, so a non-integer value is reported here.
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/** What a fetched novel cover actually is. */
|
|
2
|
+
export type NovelCoverType = 'custom' | 'pixiv_generated' | 'unknown';
|
|
3
|
+
/**
|
|
4
|
+
* The canvas Pixiv renders its built-in novel cover designs on. Every design
|
|
5
|
+
* observed in production (floral, seasonal sweets, treasure map, genre label,
|
|
6
|
+
* city night) is delivered at exactly this size, while author covers keep their
|
|
7
|
+
* own dimensions (512x512, 768x768, 800x1200, 822x1200, 826x1169, 1024x1024 …).
|
|
8
|
+
*/
|
|
9
|
+
export declare const PIXIV_GENERATED_COVER_WIDTH = 640;
|
|
10
|
+
export declare const PIXIV_GENERATED_COVER_HEIGHT = 900;
|
|
11
|
+
export interface NovelCoverPolicy {
|
|
12
|
+
/**
|
|
13
|
+
* What to do with a cover whose content type could not be classified.
|
|
14
|
+
* 'skip' (default): safe mode — never ship an unclassifiable cover.
|
|
15
|
+
* 'keep': prefer availability over certainty.
|
|
16
|
+
*/
|
|
17
|
+
unknownCover: 'skip' | 'keep';
|
|
18
|
+
}
|
|
19
|
+
/** Production-safe default: an unclassifiable cover is never shipped. */
|
|
20
|
+
export declare const DEFAULT_NOVEL_COVER_POLICY: NovelCoverPolicy;
|
|
21
|
+
/**
|
|
22
|
+
* Classify fetched cover bytes. Returns `unknown` when the image header cannot
|
|
23
|
+
* be read (no decoder in this project, so only the header is inspected).
|
|
24
|
+
*/
|
|
25
|
+
export declare function classifyNovelCover(cover: ArrayBuffer | Uint8Array | null | undefined): NovelCoverType;
|
|
26
|
+
/** The delivery decision for one classified cover under a policy. */
|
|
27
|
+
export declare function coverDeliveryDecision(policy: NovelCoverPolicy, type: NovelCoverType): 'deliver' | 'skip';
|
|
28
|
+
//# sourceMappingURL=NovelCoverPolicy.d.ts.map
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.DEFAULT_NOVEL_COVER_POLICY = exports.PIXIV_GENERATED_COVER_HEIGHT = exports.PIXIV_GENERATED_COVER_WIDTH = void 0;
|
|
4
|
+
exports.classifyNovelCover = classifyNovelCover;
|
|
5
|
+
exports.coverDeliveryDecision = coverDeliveryDecision;
|
|
6
|
+
/**
|
|
7
|
+
* Novel cover content policy (§media-asset-pipeline / §novel-cover).
|
|
8
|
+
*
|
|
9
|
+
* Pixiv exposes a novel cover as a single URL and gives NO field that says what
|
|
10
|
+
* the image actually IS. Two very different things arrive through that one
|
|
11
|
+
* field:
|
|
12
|
+
*
|
|
13
|
+
* custom the author's own uploaded artwork — real content, must ship
|
|
14
|
+
* pixiv_generated Pixiv's built-in design cover, rendered per novel on a fixed
|
|
15
|
+
* 640x900 canvas with the title typeset on it — zero content
|
|
16
|
+
* value, must never ship as Telegram media
|
|
17
|
+
*
|
|
18
|
+
* The URL cannot separate them (both live under `novel-cover-master/img/...`
|
|
19
|
+
* behind a unique hash, and the old `novel-cover-master-default` placeholder no
|
|
20
|
+
* longer occurs), so the classification is made from the fetched bytes' header.
|
|
21
|
+
* A third outcome exists on purpose:
|
|
22
|
+
*
|
|
23
|
+
* unknown the payload could not be classified (unrecognized header,
|
|
24
|
+
* format Pixiv does not normally serve)
|
|
25
|
+
*
|
|
26
|
+
* `unknown` is a POLICY decision, not an accident: sending it can leak a design
|
|
27
|
+
* cover into a public channel (the exact bug this pipeline fixes), while
|
|
28
|
+
* dropping it costs one illustration on the card. The production-safe default is
|
|
29
|
+
* therefore to skip it and log `coverType=unknown` loudly, so a future Pixiv
|
|
30
|
+
* cover-format change surfaces as a visible log line instead of silently
|
|
31
|
+
* shipping generated covers again.
|
|
32
|
+
*
|
|
33
|
+
* A FAILED probe (network / auth / rate limit) is deliberately NOT `unknown`:
|
|
34
|
+
* it is `probe_failed` and always keeps the cover, because a transient fetch
|
|
35
|
+
* error must never cost a real one.
|
|
36
|
+
*/
|
|
37
|
+
const imageDimensions_1 = require("../../utils/imageDimensions");
|
|
38
|
+
/**
|
|
39
|
+
* The canvas Pixiv renders its built-in novel cover designs on. Every design
|
|
40
|
+
* observed in production (floral, seasonal sweets, treasure map, genre label,
|
|
41
|
+
* city night) is delivered at exactly this size, while author covers keep their
|
|
42
|
+
* own dimensions (512x512, 768x768, 800x1200, 822x1200, 826x1169, 1024x1024 …).
|
|
43
|
+
*/
|
|
44
|
+
exports.PIXIV_GENERATED_COVER_WIDTH = 640;
|
|
45
|
+
exports.PIXIV_GENERATED_COVER_HEIGHT = 900;
|
|
46
|
+
/** Production-safe default: an unclassifiable cover is never shipped. */
|
|
47
|
+
exports.DEFAULT_NOVEL_COVER_POLICY = { unknownCover: 'skip' };
|
|
48
|
+
/**
|
|
49
|
+
* Classify fetched cover bytes. Returns `unknown` when the image header cannot
|
|
50
|
+
* be read (no decoder in this project, so only the header is inspected).
|
|
51
|
+
*/
|
|
52
|
+
function classifyNovelCover(cover) {
|
|
53
|
+
const dimensions = (0, imageDimensions_1.readImageDimensions)(cover);
|
|
54
|
+
if (!dimensions)
|
|
55
|
+
return 'unknown';
|
|
56
|
+
if (dimensions.width === exports.PIXIV_GENERATED_COVER_WIDTH &&
|
|
57
|
+
dimensions.height === exports.PIXIV_GENERATED_COVER_HEIGHT) {
|
|
58
|
+
return 'pixiv_generated';
|
|
59
|
+
}
|
|
60
|
+
return 'custom';
|
|
61
|
+
}
|
|
62
|
+
/** The delivery decision for one classified cover under a policy. */
|
|
63
|
+
function coverDeliveryDecision(policy, type) {
|
|
64
|
+
if (type === 'pixiv_generated')
|
|
65
|
+
return 'skip';
|
|
66
|
+
if (type === 'unknown')
|
|
67
|
+
return policy.unknownCover === 'keep' ? 'deliver' : 'skip';
|
|
68
|
+
return 'deliver';
|
|
69
|
+
}
|
|
70
|
+
//# sourceMappingURL=NovelCoverPolicy.js.map
|
|
@@ -6,6 +6,7 @@ const RankingService_1 = require("./RankingService");
|
|
|
6
6
|
const IllustrationDownloader_1 = require("./IllustrationDownloader");
|
|
7
7
|
const NovelDownloader_1 = require("./NovelDownloader");
|
|
8
8
|
const MaterializationPolicy_1 = require("../domain/media/MaterializationPolicy");
|
|
9
|
+
const NovelCoverPolicy_1 = require("../domain/media/NovelCoverPolicy");
|
|
9
10
|
const ProgressReporter_1 = require("./report/ProgressReporter");
|
|
10
11
|
const DownloadPlanner_1 = require("./plan/DownloadPlanner");
|
|
11
12
|
const DownloadExecutor_1 = require("./exec/DownloadExecutor");
|
|
@@ -129,7 +130,10 @@ class DownloadManager {
|
|
|
129
130
|
const materialization = config.download?.materializationPolicy
|
|
130
131
|
? { mode: config.download?.materializationPolicy }
|
|
131
132
|
: MaterializationPolicy_1.DEFAULT_MATERIALIZATION_POLICY;
|
|
132
|
-
|
|
133
|
+
const novelCoverPolicy = {
|
|
134
|
+
unknownCover: config.download?.novelCover?.unknown ?? NovelCoverPolicy_1.DEFAULT_NOVEL_COVER_POLICY.unknownCover,
|
|
135
|
+
};
|
|
136
|
+
this.novelDownloader = new NovelDownloader_1.NovelDownloader(client, database, fileService, database, undefined, materialization, novelCoverPolicy);
|
|
133
137
|
this.planner = new DownloadPlanner_1.DownloadPlanner(database, {
|
|
134
138
|
// `scope` is a single legacy target name, or the full fan-out array for a
|
|
135
139
|
// multi-platform target (then only works confirmed on EVERY platform count).
|
|
@@ -6,6 +6,7 @@ import { PixivNovel } from '@redtidev/pixiv-client';
|
|
|
6
6
|
import { DownloadedArtifact } from '../delivery/types';
|
|
7
7
|
import { type MaterializationPolicy } from '../domain/media/MaterializationPolicy';
|
|
8
8
|
import { type MediaMaterializer } from './materialization/MediaMaterializer';
|
|
9
|
+
import { type NovelCoverPolicy } from '../domain/media/NovelCoverPolicy';
|
|
9
10
|
import type { Database } from '../storage/Database';
|
|
10
11
|
export declare class NovelDownloader {
|
|
11
12
|
private readonly client;
|
|
@@ -14,7 +15,28 @@ export declare class NovelDownloader {
|
|
|
14
15
|
private readonly metadataDb?;
|
|
15
16
|
private readonly materializer;
|
|
16
17
|
private readonly materializationPolicy;
|
|
17
|
-
|
|
18
|
+
private readonly novelCoverPolicy;
|
|
19
|
+
constructor(client: IPixivClient, database: IDatabase, fileService: IFileService, metadataDb?: Database | undefined, materializer?: MediaMaterializer, materializationPolicy?: MaterializationPolicy, novelCoverPolicy?: NovelCoverPolicy);
|
|
18
20
|
download(novel: PixivNovel, tag: string, target: TargetConfig): Promise<DownloadedArtifact | undefined>;
|
|
21
|
+
/**
|
|
22
|
+
* Resolves the cover URL a novel should ship with (§novel-cover).
|
|
23
|
+
*
|
|
24
|
+
* Pixiv's "author set no cover" case is invisible in the API: the design it
|
|
25
|
+
* renders (title typeset on a template) is served from the same CDN path with
|
|
26
|
+
* a unique hash as a real cover, so the URL cannot decide. The candidate cover
|
|
27
|
+
* is therefore fetched once and classified from its header; Pixiv's design
|
|
28
|
+
* canvas is exactly 640x900.
|
|
29
|
+
*
|
|
30
|
+
* Policy (§media-asset-pipeline):
|
|
31
|
+
* - custom → deliver the cover
|
|
32
|
+
* - pixiv_generated → never deliver (a design cover is not content and must
|
|
33
|
+
* not become Telegram media)
|
|
34
|
+
* - unknown → policy decision, default 'skip' (safe mode), so a future
|
|
35
|
+
* Pixiv cover-format change surfaces as a loud
|
|
36
|
+
* `coverType=unknown` log instead of leaking silently
|
|
37
|
+
* A FAILED probe (network / auth / rate limit) is a separate case and always
|
|
38
|
+
* keeps the cover: a transient fetch error must never cost a real one.
|
|
39
|
+
*/
|
|
40
|
+
private resolveCoverUrl;
|
|
19
41
|
}
|
|
20
42
|
//# sourceMappingURL=NovelDownloader.d.ts.map
|
|
@@ -43,6 +43,7 @@ const Artifact_1 = require("../domain/media/Artifact");
|
|
|
43
43
|
const MediaMaterializer_1 = require("./materialization/MediaMaterializer");
|
|
44
44
|
const novelMarkers_1 = require("./novelMarkers");
|
|
45
45
|
const novelCover_1 = require("./novelCover");
|
|
46
|
+
const NovelCoverPolicy_1 = require("../domain/media/NovelCoverPolicy");
|
|
46
47
|
const zip_1 = require("../utils/zip");
|
|
47
48
|
const LANGUAGE_CACHE_TTL_MS = 7 * 24 * 60 * 60 * 1000;
|
|
48
49
|
class NovelDownloader {
|
|
@@ -52,13 +53,15 @@ class NovelDownloader {
|
|
|
52
53
|
metadataDb;
|
|
53
54
|
materializer;
|
|
54
55
|
materializationPolicy;
|
|
55
|
-
|
|
56
|
+
novelCoverPolicy;
|
|
57
|
+
constructor(client, database, fileService, metadataDb, materializer, materializationPolicy = MaterializationPolicy_1.DEFAULT_MATERIALIZATION_POLICY, novelCoverPolicy = NovelCoverPolicy_1.DEFAULT_NOVEL_COVER_POLICY) {
|
|
56
58
|
this.client = client;
|
|
57
59
|
this.database = database;
|
|
58
60
|
this.fileService = fileService;
|
|
59
61
|
this.metadataDb = metadataDb;
|
|
60
62
|
this.materializer = materializer ?? new MediaMaterializer_1.PixivMediaMaterializer(client, fileService);
|
|
61
63
|
this.materializationPolicy = materializationPolicy;
|
|
64
|
+
this.novelCoverPolicy = novelCoverPolicy;
|
|
62
65
|
}
|
|
63
66
|
async download(novel, tag, target) {
|
|
64
67
|
// Metadata cache: language filtering otherwise pulls FULL text per candidate
|
|
@@ -226,10 +229,11 @@ class NovelDownloader {
|
|
|
226
229
|
}
|
|
227
230
|
}
|
|
228
231
|
// Novel cover (§novel-cover): normalized to null for Pixiv's default
|
|
229
|
-
// placeholder
|
|
230
|
-
//
|
|
231
|
-
// cover
|
|
232
|
-
|
|
232
|
+
// placeholder AND for Pixiv's generated design covers, carried as a
|
|
233
|
+
// dedicated `novelcover` MediaAsset so TelePost can build
|
|
234
|
+
// `cover root + TXT reply` without positional guessing. The cover is never
|
|
235
|
+
// materialized locally — Telegram fetches it (via proxy).
|
|
236
|
+
const coverUrl = await this.resolveCoverUrl(detail.id, typeof textResponse === 'string' ? undefined : textResponse.coverUrl);
|
|
233
237
|
const coverAsset = coverUrl ? (0, novelCover_1.novelCoverAsset)(String(detail.id), coverUrl) : undefined;
|
|
234
238
|
const downloadByAssetKey = new Map(assets.filter((a) => a.status === 'downloaded' && a.localPath)
|
|
235
239
|
.map((a) => [`${a.kind}:${a.sourceId}`, a]));
|
|
@@ -398,6 +402,66 @@ class NovelDownloader {
|
|
|
398
402
|
language: detectedLang ? `${detectedLang.name} (${detectedLang.code})` : undefined,
|
|
399
403
|
};
|
|
400
404
|
}
|
|
405
|
+
/**
|
|
406
|
+
* Resolves the cover URL a novel should ship with (§novel-cover).
|
|
407
|
+
*
|
|
408
|
+
* Pixiv's "author set no cover" case is invisible in the API: the design it
|
|
409
|
+
* renders (title typeset on a template) is served from the same CDN path with
|
|
410
|
+
* a unique hash as a real cover, so the URL cannot decide. The candidate cover
|
|
411
|
+
* is therefore fetched once and classified from its header; Pixiv's design
|
|
412
|
+
* canvas is exactly 640x900.
|
|
413
|
+
*
|
|
414
|
+
* Policy (§media-asset-pipeline):
|
|
415
|
+
* - custom → deliver the cover
|
|
416
|
+
* - pixiv_generated → never deliver (a design cover is not content and must
|
|
417
|
+
* not become Telegram media)
|
|
418
|
+
* - unknown → policy decision, default 'skip' (safe mode), so a future
|
|
419
|
+
* Pixiv cover-format change surfaces as a loud
|
|
420
|
+
* `coverType=unknown` log instead of leaking silently
|
|
421
|
+
* A FAILED probe (network / auth / rate limit) is a separate case and always
|
|
422
|
+
* keeps the cover: a transient fetch error must never cost a real one.
|
|
423
|
+
*/
|
|
424
|
+
async resolveCoverUrl(novelId, coverUrl) {
|
|
425
|
+
const normalized = (0, novelCover_1.normalizeNovelCoverUrl)(coverUrl);
|
|
426
|
+
if (!normalized)
|
|
427
|
+
return null;
|
|
428
|
+
let cover;
|
|
429
|
+
try {
|
|
430
|
+
cover = await this.client.downloadImage(normalized);
|
|
431
|
+
}
|
|
432
|
+
catch (error) {
|
|
433
|
+
logger_1.logger.warn(`Novel ${novelId} cover probe failed; keeping the cover (coverType=probe_failed)`, {
|
|
434
|
+
novelId,
|
|
435
|
+
coverUrl: normalized,
|
|
436
|
+
coverType: 'probe_failed',
|
|
437
|
+
reason: error instanceof Error ? error.message : String(error),
|
|
438
|
+
});
|
|
439
|
+
return normalized;
|
|
440
|
+
}
|
|
441
|
+
const coverType = (0, NovelCoverPolicy_1.classifyNovelCover)(cover);
|
|
442
|
+
const decision = (0, NovelCoverPolicy_1.coverDeliveryDecision)(this.novelCoverPolicy, coverType);
|
|
443
|
+
const canvas = `${NovelCoverPolicy_1.PIXIV_GENERATED_COVER_WIDTH}x${NovelCoverPolicy_1.PIXIV_GENERATED_COVER_HEIGHT}`;
|
|
444
|
+
if (decision === 'skip') {
|
|
445
|
+
if (coverType === 'pixiv_generated') {
|
|
446
|
+
logger_1.logger.info(`Novel ${novelId} cover is a Pixiv generated design (${canvas}); delivering without a cover (coverType=${coverType})`, { novelId, coverUrl: normalized, coverType, canvas });
|
|
447
|
+
}
|
|
448
|
+
else {
|
|
449
|
+
logger_1.logger.warn(`Novel ${novelId} cover could not be classified; skipping it per novelCover.unknown=skip (coverType=${coverType})`, { novelId, coverUrl: normalized, coverType, policy: this.novelCoverPolicy.unknownCover });
|
|
450
|
+
}
|
|
451
|
+
return null;
|
|
452
|
+
}
|
|
453
|
+
if (coverType === 'unknown') {
|
|
454
|
+
logger_1.logger.warn(`Novel ${novelId} cover could not be classified; keeping it per novelCover.unknown=keep (coverType=${coverType})`, { novelId, coverUrl: normalized, coverType, policy: this.novelCoverPolicy.unknownCover });
|
|
455
|
+
}
|
|
456
|
+
else {
|
|
457
|
+
logger_1.logger.debug(`Novel ${novelId} cover classified (coverType=${coverType})`, {
|
|
458
|
+
novelId,
|
|
459
|
+
coverUrl: normalized,
|
|
460
|
+
coverType,
|
|
461
|
+
});
|
|
462
|
+
}
|
|
463
|
+
return normalized;
|
|
464
|
+
}
|
|
401
465
|
}
|
|
402
466
|
exports.NovelDownloader = NovelDownloader;
|
|
403
467
|
function buildMediaAssetById(mediaAsset, artifactIdValue) {
|
|
@@ -2,12 +2,23 @@
|
|
|
2
2
|
* Novel cover normalization (§novel-cover).
|
|
3
3
|
*
|
|
4
4
|
* Pixiv exposes the novel cover as `coverUrl` on the webview v2 text response.
|
|
5
|
-
* Novels WITHOUT a custom cover return the stock
|
|
6
|
-
* placeholder
|
|
7
|
-
*
|
|
5
|
+
* Novels WITHOUT a custom cover used to return the stock
|
|
6
|
+
* "novel-cover-master-default" placeholder; today Pixiv instead RENDERS a
|
|
7
|
+
* per-novel design (floral / seasonal / genre template with the novel title
|
|
8
|
+
* typeset) and serves it from `novel-cover-master/img/...` with a unique hash,
|
|
9
|
+
* so it is URL-indistinguishable from an author cover. Those design covers are
|
|
10
|
+
* NOT real covers and must never ship as Telegram media, and no API field
|
|
11
|
+
* (app-api `novel/detail`, webview v2) marks them: the one reliable
|
|
12
|
+
* discriminator is the canvas Pixiv renders them on, always exactly 640x900
|
|
13
|
+
* (`PIXIV_DESIGN_COVER_*` below). The normalized model is therefore
|
|
14
|
+
* `string | null`:
|
|
8
15
|
*
|
|
9
|
-
* real cover
|
|
10
|
-
* default / absent → null
|
|
16
|
+
* real cover → the Pixiv CDN URL (original size when derivable)
|
|
17
|
+
* default / design / absent → null
|
|
18
|
+
*
|
|
19
|
+
* Callers cannot tell a design cover from the URL alone, so the fetch-and-check
|
|
20
|
+
* step (`isPixivDesignCoverImage`) lives in the downloader; this module stays a
|
|
21
|
+
* pure URL/bytes helper.
|
|
11
22
|
*
|
|
12
23
|
* The cover rides the canonical MediaAsset contract with a dedicated
|
|
13
24
|
* `novelcover` pixivKind, so consumers (TelePost) can distinguish it from
|
|
@@ -15,6 +26,19 @@
|
|
|
15
26
|
* positional guessing.
|
|
16
27
|
*/
|
|
17
28
|
import { MediaAsset } from '../domain/media/MediaAsset';
|
|
29
|
+
/**
|
|
30
|
+
* Historical names for the generated-cover canvas; the classification and the
|
|
31
|
+
* policy live in `src/domain/media/NovelCoverPolicy.ts` (§media-asset-pipeline).
|
|
32
|
+
*/
|
|
33
|
+
export declare const PIXIV_DESIGN_COVER_WIDTH = 640;
|
|
34
|
+
export declare const PIXIV_DESIGN_COVER_HEIGHT = 900;
|
|
35
|
+
/**
|
|
36
|
+
* True when the fetched cover bytes are one of Pixiv's generated designs.
|
|
37
|
+
* Unknown formats / unreadable payloads return false — this predicate is the
|
|
38
|
+
* raw classification only; whether an unclassifiable cover is delivered is the
|
|
39
|
+
* caller's policy (`coverDeliveryDecision`, default: skipped).
|
|
40
|
+
*/
|
|
41
|
+
export declare function isPixivDesignCoverImage(cover: ArrayBuffer | Uint8Array | null | undefined): boolean;
|
|
18
42
|
export declare function normalizeNovelCoverUrl(coverUrl?: string | null): string | null;
|
|
19
43
|
export declare function novelCoverAsset(workId: string, coverUrl: string): MediaAsset;
|
|
20
44
|
//# sourceMappingURL=novelCover.d.ts.map
|
|
@@ -1,17 +1,30 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.PIXIV_DESIGN_COVER_HEIGHT = exports.PIXIV_DESIGN_COVER_WIDTH = void 0;
|
|
4
|
+
exports.isPixivDesignCoverImage = isPixivDesignCoverImage;
|
|
3
5
|
exports.normalizeNovelCoverUrl = normalizeNovelCoverUrl;
|
|
4
6
|
exports.novelCoverAsset = novelCoverAsset;
|
|
5
7
|
/**
|
|
6
8
|
* Novel cover normalization (§novel-cover).
|
|
7
9
|
*
|
|
8
10
|
* Pixiv exposes the novel cover as `coverUrl` on the webview v2 text response.
|
|
9
|
-
* Novels WITHOUT a custom cover return the stock
|
|
10
|
-
* placeholder
|
|
11
|
-
*
|
|
11
|
+
* Novels WITHOUT a custom cover used to return the stock
|
|
12
|
+
* "novel-cover-master-default" placeholder; today Pixiv instead RENDERS a
|
|
13
|
+
* per-novel design (floral / seasonal / genre template with the novel title
|
|
14
|
+
* typeset) and serves it from `novel-cover-master/img/...` with a unique hash,
|
|
15
|
+
* so it is URL-indistinguishable from an author cover. Those design covers are
|
|
16
|
+
* NOT real covers and must never ship as Telegram media, and no API field
|
|
17
|
+
* (app-api `novel/detail`, webview v2) marks them: the one reliable
|
|
18
|
+
* discriminator is the canvas Pixiv renders them on, always exactly 640x900
|
|
19
|
+
* (`PIXIV_DESIGN_COVER_*` below). The normalized model is therefore
|
|
20
|
+
* `string | null`:
|
|
12
21
|
*
|
|
13
|
-
* real cover
|
|
14
|
-
* default / absent → null
|
|
22
|
+
* real cover → the Pixiv CDN URL (original size when derivable)
|
|
23
|
+
* default / design / absent → null
|
|
24
|
+
*
|
|
25
|
+
* Callers cannot tell a design cover from the URL alone, so the fetch-and-check
|
|
26
|
+
* step (`isPixivDesignCoverImage`) lives in the downloader; this module stays a
|
|
27
|
+
* pure URL/bytes helper.
|
|
15
28
|
*
|
|
16
29
|
* The cover rides the canonical MediaAsset contract with a dedicated
|
|
17
30
|
* `novelcover` pixivKind, so consumers (TelePost) can distinguish it from
|
|
@@ -19,9 +32,25 @@ exports.novelCoverAsset = novelCoverAsset;
|
|
|
19
32
|
* positional guessing.
|
|
20
33
|
*/
|
|
21
34
|
const MediaAsset_1 = require("../domain/media/MediaAsset");
|
|
35
|
+
const NovelCoverPolicy_1 = require("../domain/media/NovelCoverPolicy");
|
|
22
36
|
const DEFAULT_COVER_PATTERN = /novel-cover-(master-)?default/i;
|
|
23
37
|
/** Strip the `/c/<spec>/` resizer segment to recover the original-size URL. */
|
|
24
38
|
const RESIZED_COVER_PATTERN = /\/c\/[^/]+\/(novel-cover-master\/)/;
|
|
39
|
+
/**
|
|
40
|
+
* Historical names for the generated-cover canvas; the classification and the
|
|
41
|
+
* policy live in `src/domain/media/NovelCoverPolicy.ts` (§media-asset-pipeline).
|
|
42
|
+
*/
|
|
43
|
+
exports.PIXIV_DESIGN_COVER_WIDTH = NovelCoverPolicy_1.PIXIV_GENERATED_COVER_WIDTH;
|
|
44
|
+
exports.PIXIV_DESIGN_COVER_HEIGHT = NovelCoverPolicy_1.PIXIV_GENERATED_COVER_HEIGHT;
|
|
45
|
+
/**
|
|
46
|
+
* True when the fetched cover bytes are one of Pixiv's generated designs.
|
|
47
|
+
* Unknown formats / unreadable payloads return false — this predicate is the
|
|
48
|
+
* raw classification only; whether an unclassifiable cover is delivered is the
|
|
49
|
+
* caller's policy (`coverDeliveryDecision`, default: skipped).
|
|
50
|
+
*/
|
|
51
|
+
function isPixivDesignCoverImage(cover) {
|
|
52
|
+
return (0, NovelCoverPolicy_1.classifyNovelCover)(cover) === 'pixiv_generated';
|
|
53
|
+
}
|
|
25
54
|
function normalizeNovelCoverUrl(coverUrl) {
|
|
26
55
|
const url = typeof coverUrl === 'string' ? coverUrl.trim() : '';
|
|
27
56
|
if (!url)
|