pixivflow 3.0.3 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/TopicCommand.js +41 -9
- package/dist/config/types.d.ts +54 -0
- package/dist/config/types.js +3 -0
- package/dist/config/validation.js +44 -0
- package/dist/domain/media/NovelCoverPolicy.d.ts +28 -0
- package/dist/domain/media/NovelCoverPolicy.js +70 -0
- package/dist/download/DownloadManager.js +5 -1
- package/dist/download/NovelDownloader.d.ts +15 -4
- package/dist/download/NovelDownloader.js +43 -11
- package/dist/download/novelCover.d.ts +5 -6
- package/dist/download/novelCover.js +9 -12
- package/dist/package.json +1 -1
- package/dist/topic/TopicPipeline.d.ts +38 -7
- package/dist/topic/TopicPipeline.js +177 -30
- package/dist/topic/TopicResolver.js +4 -1
- package/dist/topic/TopicTagScorer.js +12 -1
- package/dist/topic/types.d.ts +48 -0
- package/dist/version.js +1 -1
- package/dist/webui/package.json +1 -1
- package/examples/gateway/README.md +4 -0
- package/examples/onebot-adapter/README.md +139 -0
- package/examples/onebot-adapter/server.mjs +612 -0
- package/package.json +2 -1
|
@@ -7,6 +7,7 @@ const Database_1 = require("../storage/Database");
|
|
|
7
7
|
const PixivAuth_1 = require("../auth/PixivAuth");
|
|
8
8
|
const createPixivFlowClient_1 = require("../pixiv-client/createPixivFlowClient");
|
|
9
9
|
const createTopicPipeline_1 = require("../topic/createTopicPipeline");
|
|
10
|
+
const TopicPipeline_1 = require("../topic/TopicPipeline");
|
|
10
11
|
const TopicResolver_1 = require("../topic/TopicResolver");
|
|
11
12
|
const TopicCache_1 = require("../topic/TopicCache");
|
|
12
13
|
const node_path_1 = require("node:path");
|
|
@@ -50,27 +51,48 @@ class TopicCommand extends Command_1.BaseCommand {
|
|
|
50
51
|
database.close();
|
|
51
52
|
}
|
|
52
53
|
}
|
|
53
|
-
async resolve(context,
|
|
54
|
+
async resolve(context, args, topic) {
|
|
54
55
|
if (!topic) {
|
|
55
56
|
console.error('\nUsage: pixivflow topic resolve <topic> [--type illustration|novel|all] [--refresh]\n');
|
|
56
57
|
return this.failure('Missing topic for resolve');
|
|
57
58
|
}
|
|
58
|
-
const type = (String(
|
|
59
|
-
const refresh = Boolean(
|
|
59
|
+
const type = (String(args.options.type ?? 'all'));
|
|
60
|
+
const refresh = Boolean(args.options.refresh);
|
|
60
61
|
const types = type === 'all' ? ['illustration', 'novel'] : [type];
|
|
61
62
|
return this.withClient(context, async ({ client, database }) => {
|
|
62
63
|
const cache = new TopicCache_1.TopicCache((0, node_path_1.dirname)(database.getDatabasePath()) + '/topic-cache');
|
|
64
|
+
const discovery = context.config.targets?.find((target) => target.mode === 'topic')?.topicDiscovery ?? {};
|
|
65
|
+
// §tag-provenance: the audit surface for "the original tag dominates and no
|
|
66
|
+
// related tag outranks it". Each row exposes where the tag came from, its
|
|
67
|
+
// semantic weight and whether the current tagRelations/relatedTags settings
|
|
68
|
+
// would actually search it today (the `searched` column).
|
|
69
|
+
const rows = [];
|
|
70
|
+
const lines = [];
|
|
63
71
|
for (const contentType of types) {
|
|
64
72
|
const resolver = new TopicResolver_1.TopicResolver(client, cache, context.config.download?.requestDelay ?? 500);
|
|
65
73
|
const { space, fromCache, degraded } = await resolver.resolve(topic, contentType, { refresh });
|
|
66
|
-
|
|
67
|
-
|
|
74
|
+
const walked = (0, TopicPipeline_1.selectWalkedTags)(space.tags, key(topic), discovery.tagRelations);
|
|
75
|
+
const channels = new Set((0, TopicPipeline_1.recallChannels)(walked, key(topic), relatedMode(discovery.relatedTags)).map((tag) => key(tag.name)));
|
|
76
|
+
lines.push('');
|
|
77
|
+
lines.push(`Topic: ${topic} (${contentType}) ${fromCache ? (degraded ? '· stale cache' : '· cache') : '· fresh'}`);
|
|
78
|
+
lines.push('Name'.padEnd(24) + 'Trans'.padEnd(16) + 'Source'.padEnd(28) + 'Weight'.padStart(7) + 'Score'.padStart(7) + 'Seed'.padStart(6) + 'Searched'.padStart(10));
|
|
79
|
+
const tagRows = [];
|
|
68
80
|
for (const tag of space.tags) {
|
|
69
|
-
const
|
|
70
|
-
|
|
81
|
+
const source = tag.source ?? (tag.seed ? 'seed' : 'cooccurrence');
|
|
82
|
+
const weight = tag.weight ?? tag.score;
|
|
83
|
+
const searched = channels.has(key(tag.name));
|
|
84
|
+
tagRows.push({ name: tag.name, translatedName: tag.translatedName, source, weight, score: tag.score, seed: tag.seed, searched });
|
|
85
|
+
lines.push(tag.name.slice(0, 23).padEnd(24)
|
|
86
|
+
+ (tag.translatedName ?? '-').slice(0, 15).padEnd(16)
|
|
87
|
+
+ source.padEnd(28)
|
|
88
|
+
+ weight.toFixed(2).padStart(7)
|
|
89
|
+
+ tag.score.toFixed(2).padStart(7)
|
|
90
|
+
+ (tag.seed ? 'yes' : 'no').padStart(6)
|
|
91
|
+
+ (searched ? 'yes' : 'no').padStart(10));
|
|
71
92
|
}
|
|
93
|
+
rows.push({ contentType, fromCache, degraded, tags: tagRows });
|
|
72
94
|
}
|
|
73
|
-
return this.success('
|
|
95
|
+
return this.success(lines.join('\n'), { topic, types: rows });
|
|
74
96
|
});
|
|
75
97
|
}
|
|
76
98
|
async test(context, args, topic) {
|
|
@@ -92,6 +114,7 @@ class TopicCommand extends Command_1.BaseCommand {
|
|
|
92
114
|
const { works, selection } = await pipeline.selectWorks(target, contentType, day, limit, { refresh }, {});
|
|
93
115
|
console.log(`\n=== ${contentType} topic "${topic}" day ${day} ===`);
|
|
94
116
|
console.log(`resolvedTags=${selection.resolvedTagCount} raw=${selection.rawCount} deduped=${selection.dedupedCount} accepted=${selection.acceptedCount} selected=${works.length}`);
|
|
117
|
+
console.log(`searchedTags=${(selection.searchedTags ?? []).join(', ')}`);
|
|
95
118
|
selection.selected.forEach((c, i) => {
|
|
96
119
|
console.log(` #${i + 1} id=${c.id} pop=${c.popularity.toFixed(1)} meta=${c.metadataScore.toFixed(2)} ${c.title}`);
|
|
97
120
|
});
|
|
@@ -103,7 +126,8 @@ class TopicCommand extends Command_1.BaseCommand {
|
|
|
103
126
|
getUsage() {
|
|
104
127
|
return [
|
|
105
128
|
'topic resolve <topic> [--type all|illustration|novel] [--refresh]',
|
|
106
|
-
' Show the Pixiv-derived related tag space for a topic (cached)
|
|
129
|
+
' Show the Pixiv-derived related tag space for a topic (cached), with each',
|
|
130
|
+
' tag\'s provenance (source), semantic weight, score and whether it is searched.',
|
|
107
131
|
'',
|
|
108
132
|
'topic test <topic> [--type all|illustration|novel] [--date YESTERDAY|YYYY-MM-DD] [--limit N] [--refresh]',
|
|
109
133
|
' Dry-run a daily selection: resolved tags, candidate counts, Top N (no downloads).',
|
|
@@ -111,4 +135,12 @@ class TopicCommand extends Command_1.BaseCommand {
|
|
|
111
135
|
}
|
|
112
136
|
}
|
|
113
137
|
exports.TopicCommand = TopicCommand;
|
|
138
|
+
/** Same normalization as the pipeline/resolver keys (trim + NFKC + lowercase). */
|
|
139
|
+
function key(value) {
|
|
140
|
+
return value.normalize('NFKC').trim().toLocaleLowerCase();
|
|
141
|
+
}
|
|
142
|
+
/** Unknown modes fall back to the historical 'always', as in the pipeline. */
|
|
143
|
+
function relatedMode(value) {
|
|
144
|
+
return value === 'when_seed_insufficient' || value === 'never' ? value : 'always';
|
|
145
|
+
}
|
|
114
146
|
//# sourceMappingURL=TopicCommand.js.map
|
package/dist/config/types.d.ts
CHANGED
|
@@ -4,6 +4,25 @@
|
|
|
4
4
|
export type TargetType = 'illustration' | 'novel';
|
|
5
5
|
import type { DeliveryCapabilityOverrides } from '../delivery/capabilities';
|
|
6
6
|
export type DeliveryFieldValue = string | number | boolean | string[];
|
|
7
|
+
/** Provenance names accepted by `TopicDiscoveryConfig.tagRelations`. */
|
|
8
|
+
export declare const TAG_RELATION_SOURCES: readonly ["seed", "cooccurrence", "autocomplete"];
|
|
9
|
+
export type TagRelationSource = (typeof TAG_RELATION_SOURCES)[number];
|
|
10
|
+
/**
|
|
11
|
+
* Which resolved tags may become recall channels (§tag-provenance). Every list
|
|
12
|
+
* is optional and the defaults reproduce the pre-existing behaviour: walk the
|
|
13
|
+
* whole resolved space.
|
|
14
|
+
*
|
|
15
|
+
* `deny` always wins, then `allowSources`, then `allow`. The seed tag is never
|
|
16
|
+
* dropped by `allow`/`allowSources` — only `deny` can drop it.
|
|
17
|
+
*/
|
|
18
|
+
export interface TagRelationsConfig {
|
|
19
|
+
/** Provenance categories allowed to be walked (default: all three). */
|
|
20
|
+
allowSources?: TagRelationSource[];
|
|
21
|
+
/** If non-empty: ONLY these tag names are walked (the seed tag is kept). */
|
|
22
|
+
allow?: string[];
|
|
23
|
+
/** Always dropped, wins over `allow`/`allowSources` (default: none). */
|
|
24
|
+
deny?: string[];
|
|
25
|
+
}
|
|
7
26
|
/** Discovery tuning for mode='topic'. Every value has a safe default. */
|
|
8
27
|
export interface TopicDiscoveryConfig {
|
|
9
28
|
/** Max related tags used to build the search space (default 12). */
|
|
@@ -28,6 +47,26 @@ export interface TopicDiscoveryConfig {
|
|
|
28
47
|
* - `'never'`: search the topic tag alone.
|
|
29
48
|
*/
|
|
30
49
|
relatedTags?: 'always' | 'when_seed_insufficient' | 'never';
|
|
50
|
+
/**
|
|
51
|
+
* Make the seed tag a HARD ranking tier (default `'off'`).
|
|
52
|
+
*
|
|
53
|
+
* `'off'` keeps the documented popularity-only ranking, where relevance is
|
|
54
|
+
* only an acceptance gate. `'on'` guarantees the invariant "原始 Tag 权重最高,
|
|
55
|
+
* 相关 Tag 权重不得超过原始 Tag": a work carrying the seed tag always outranks
|
|
56
|
+
* a work that only matched expanded tags, however popular the latter is.
|
|
57
|
+
*/
|
|
58
|
+
seedTier?: 'off' | 'on';
|
|
59
|
+
/** Which resolved tags may be walked as recall channels (default: all). */
|
|
60
|
+
tagRelations?: TagRelationsConfig;
|
|
61
|
+
/**
|
|
62
|
+
* Also treat a work's `translated_name` as a tag hit (default false).
|
|
63
|
+
*
|
|
64
|
+
* Today only the work's `name` is compared with the resolved tag keys. When
|
|
65
|
+
* enabled, a work whose translated tag name matches a resolved tag counts as
|
|
66
|
+
* carrying that tag (and a translated match of the seed counts as a seed hit).
|
|
67
|
+
* Default `false` keeps today's matching exactly.
|
|
68
|
+
*/
|
|
69
|
+
matchTranslatedNames?: boolean;
|
|
31
70
|
}
|
|
32
71
|
/** Candidate collection tuning for mode='topic'. */
|
|
33
72
|
export interface CandidateCollectionConfig {
|
|
@@ -876,6 +915,21 @@ export interface StandaloneConfig {
|
|
|
876
915
|
* Default: 'eager'
|
|
877
916
|
*/
|
|
878
917
|
materializationPolicy?: 'eager' | 'on-demand';
|
|
918
|
+
/**
|
|
919
|
+
* Novel cover content policy (§media-asset-pipeline / §novel-cover).
|
|
920
|
+
*
|
|
921
|
+
* Pixiv serves author covers and its own generated design covers from the
|
|
922
|
+
* same URL shape, so the content type is classified from the fetched bytes:
|
|
923
|
+
* a generated design (exactly 640x900) is never delivered as Telegram media.
|
|
924
|
+
* This key governs the UNCERTAIN case only — a cover whose header cannot be
|
|
925
|
+
* classified. 'skip' (default) is safe mode: never ship an unclassifiable
|
|
926
|
+
* cover, and log `coverType=unknown` so a future Pixiv format change is
|
|
927
|
+
* visible instead of silently leaking designs again. 'keep' prefers
|
|
928
|
+
* availability over certainty. A failed probe always keeps the cover.
|
|
929
|
+
*/
|
|
930
|
+
novelCover?: {
|
|
931
|
+
unknown?: 'skip' | 'keep';
|
|
932
|
+
};
|
|
879
933
|
};
|
|
880
934
|
}
|
|
881
935
|
//# sourceMappingURL=types.d.ts.map
|
package/dist/config/types.js
CHANGED
|
@@ -3,4 +3,7 @@
|
|
|
3
3
|
* Configuration type definitions
|
|
4
4
|
*/
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.TAG_RELATION_SOURCES = void 0;
|
|
7
|
+
/** Provenance names accepted by `TopicDiscoveryConfig.tagRelations`. */
|
|
8
|
+
exports.TAG_RELATION_SOURCES = ['seed', 'cooccurrence', 'autocomplete'];
|
|
6
9
|
//# sourceMappingURL=types.js.map
|
|
@@ -194,6 +194,45 @@ function validateConfig(config, location, databasePath) {
|
|
|
194
194
|
if (td.relatedTags !== undefined && !['always', 'when_seed_insufficient', 'never'].includes(td.relatedTags)) {
|
|
195
195
|
errors.push(`targets[${index}].topicDiscovery.relatedTags: Must be "always", "when_seed_insufficient" or "never" (got ${String(td.relatedTags)})`);
|
|
196
196
|
}
|
|
197
|
+
// §tag-provenance: all optional; the defaults reproduce the pre-change
|
|
198
|
+
// behaviour (whole space walked, popularity-only ranking, name-only match).
|
|
199
|
+
if (td.seedTier !== undefined && !['off', 'on'].includes(td.seedTier)) {
|
|
200
|
+
errors.push(`targets[${index}].topicDiscovery.seedTier: Must be "off" or "on" (got ${String(td.seedTier)})`);
|
|
201
|
+
}
|
|
202
|
+
if (td.matchTranslatedNames !== undefined && typeof td.matchTranslatedNames !== 'boolean') {
|
|
203
|
+
errors.push(`targets[${index}].topicDiscovery.matchTranslatedNames: Must be a boolean (got ${typeof td.matchTranslatedNames})`);
|
|
204
|
+
}
|
|
205
|
+
const relations = td.tagRelations;
|
|
206
|
+
if (relations !== undefined) {
|
|
207
|
+
if (typeof relations !== 'object' || relations === null || Array.isArray(relations)) {
|
|
208
|
+
errors.push(`targets[${index}].topicDiscovery.tagRelations: Must be an object (got ${Array.isArray(relations) ? 'array' : typeof relations})`);
|
|
209
|
+
}
|
|
210
|
+
else {
|
|
211
|
+
const knownSources = ['seed', 'cooccurrence', 'autocomplete'];
|
|
212
|
+
const listErrors = (field) => (field === 'allowSources'
|
|
213
|
+
? `targets[${index}].topicDiscovery.tagRelations.allowSources: Must be an array of tag sources: ${knownSources.map((s) => `"${s}"`).join(', ')} (got ${JSON.stringify(relations.allowSources)})`
|
|
214
|
+
: `targets[${index}].topicDiscovery.tagRelations.${field}: Must be an array of tag names (got ${JSON.stringify(relations[field])})`);
|
|
215
|
+
const checkList = (field) => {
|
|
216
|
+
const value = relations[field];
|
|
217
|
+
if (value === undefined)
|
|
218
|
+
return undefined;
|
|
219
|
+
if (!Array.isArray(value) || value.some((entry) => typeof entry !== 'string' || entry.trim() === '')) {
|
|
220
|
+
errors.push(listErrors(field));
|
|
221
|
+
return undefined;
|
|
222
|
+
}
|
|
223
|
+
return value;
|
|
224
|
+
};
|
|
225
|
+
const allowSources = checkList('allowSources');
|
|
226
|
+
if (allowSources) {
|
|
227
|
+
const unknown = allowSources.filter((source) => !knownSources.includes(source));
|
|
228
|
+
if (unknown.length > 0) {
|
|
229
|
+
errors.push(`targets[${index}].topicDiscovery.tagRelations.allowSources: Unknown tag source${unknown.length > 1 ? 's' : ''} ${unknown.map((source) => `"${source}"`).join(', ')}; known sources are ${knownSources.map((s) => `"${s}"`).join(', ')}`);
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
checkList('allow');
|
|
233
|
+
checkList('deny');
|
|
234
|
+
}
|
|
235
|
+
}
|
|
197
236
|
}
|
|
198
237
|
const cc = target.candidateCollection;
|
|
199
238
|
if (cc) {
|
|
@@ -660,6 +699,11 @@ function validateConfig(config, location, databasePath) {
|
|
|
660
699
|
config.download.candidateScanLimit > 100)) {
|
|
661
700
|
warnings.push('download.candidateScanLimit: Should be an integer between 1 and 100');
|
|
662
701
|
}
|
|
702
|
+
if (config.download.novelCover?.unknown !== undefined &&
|
|
703
|
+
config.download.novelCover.unknown !== 'skip' &&
|
|
704
|
+
config.download.novelCover.unknown !== 'keep') {
|
|
705
|
+
errors.push(`download.novelCover.unknown: Must be "skip" or "keep" (got ${String(config.download.novelCover.unknown)})`);
|
|
706
|
+
}
|
|
663
707
|
}
|
|
664
708
|
// The bounded candidate scan is what stops a page full of duplicates from
|
|
665
709
|
// burning a whole scheduled slot, so a non-integer value is reported here.
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/** What a fetched novel cover actually is. */
|
|
2
|
+
export type NovelCoverType = 'custom' | 'pixiv_generated' | 'unknown';
|
|
3
|
+
/**
|
|
4
|
+
* The canvas Pixiv renders its built-in novel cover designs on. Every design
|
|
5
|
+
* observed in production (floral, seasonal sweets, treasure map, genre label,
|
|
6
|
+
* city night) is delivered at exactly this size, while author covers keep their
|
|
7
|
+
* own dimensions (512x512, 768x768, 800x1200, 822x1200, 826x1169, 1024x1024 …).
|
|
8
|
+
*/
|
|
9
|
+
export declare const PIXIV_GENERATED_COVER_WIDTH = 640;
|
|
10
|
+
export declare const PIXIV_GENERATED_COVER_HEIGHT = 900;
|
|
11
|
+
export interface NovelCoverPolicy {
|
|
12
|
+
/**
|
|
13
|
+
* What to do with a cover whose content type could not be classified.
|
|
14
|
+
* 'skip' (default): safe mode — never ship an unclassifiable cover.
|
|
15
|
+
* 'keep': prefer availability over certainty.
|
|
16
|
+
*/
|
|
17
|
+
unknownCover: 'skip' | 'keep';
|
|
18
|
+
}
|
|
19
|
+
/** Production-safe default: an unclassifiable cover is never shipped. */
|
|
20
|
+
export declare const DEFAULT_NOVEL_COVER_POLICY: NovelCoverPolicy;
|
|
21
|
+
/**
|
|
22
|
+
* Classify fetched cover bytes. Returns `unknown` when the image header cannot
|
|
23
|
+
* be read (no decoder in this project, so only the header is inspected).
|
|
24
|
+
*/
|
|
25
|
+
export declare function classifyNovelCover(cover: ArrayBuffer | Uint8Array | null | undefined): NovelCoverType;
|
|
26
|
+
/** The delivery decision for one classified cover under a policy. */
|
|
27
|
+
export declare function coverDeliveryDecision(policy: NovelCoverPolicy, type: NovelCoverType): 'deliver' | 'skip';
|
|
28
|
+
//# sourceMappingURL=NovelCoverPolicy.d.ts.map
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.DEFAULT_NOVEL_COVER_POLICY = exports.PIXIV_GENERATED_COVER_HEIGHT = exports.PIXIV_GENERATED_COVER_WIDTH = void 0;
|
|
4
|
+
exports.classifyNovelCover = classifyNovelCover;
|
|
5
|
+
exports.coverDeliveryDecision = coverDeliveryDecision;
|
|
6
|
+
/**
|
|
7
|
+
* Novel cover content policy (§media-asset-pipeline / §novel-cover).
|
|
8
|
+
*
|
|
9
|
+
* Pixiv exposes a novel cover as a single URL and gives NO field that says what
|
|
10
|
+
* the image actually IS. Two very different things arrive through that one
|
|
11
|
+
* field:
|
|
12
|
+
*
|
|
13
|
+
* custom the author's own uploaded artwork — real content, must ship
|
|
14
|
+
* pixiv_generated Pixiv's built-in design cover, rendered per novel on a fixed
|
|
15
|
+
* 640x900 canvas with the title typeset on it — zero content
|
|
16
|
+
* value, must never ship as Telegram media
|
|
17
|
+
*
|
|
18
|
+
* The URL cannot separate them (both live under `novel-cover-master/img/...`
|
|
19
|
+
* behind a unique hash, and the old `novel-cover-master-default` placeholder no
|
|
20
|
+
* longer occurs), so the classification is made from the fetched bytes' header.
|
|
21
|
+
* A third outcome exists on purpose:
|
|
22
|
+
*
|
|
23
|
+
* unknown the payload could not be classified (unrecognized header,
|
|
24
|
+
* format Pixiv does not normally serve)
|
|
25
|
+
*
|
|
26
|
+
* `unknown` is a POLICY decision, not an accident: sending it can leak a design
|
|
27
|
+
* cover into a public channel (the exact bug this pipeline fixes), while
|
|
28
|
+
* dropping it costs one illustration on the card. The production-safe default is
|
|
29
|
+
* therefore to skip it and log `coverType=unknown` loudly, so a future Pixiv
|
|
30
|
+
* cover-format change surfaces as a visible log line instead of silently
|
|
31
|
+
* shipping generated covers again.
|
|
32
|
+
*
|
|
33
|
+
* A FAILED probe (network / auth / rate limit) is deliberately NOT `unknown`:
|
|
34
|
+
* it is `probe_failed` and always keeps the cover, because a transient fetch
|
|
35
|
+
* error must never cost a real one.
|
|
36
|
+
*/
|
|
37
|
+
const imageDimensions_1 = require("../../utils/imageDimensions");
|
|
38
|
+
/**
|
|
39
|
+
* The canvas Pixiv renders its built-in novel cover designs on. Every design
|
|
40
|
+
* observed in production (floral, seasonal sweets, treasure map, genre label,
|
|
41
|
+
* city night) is delivered at exactly this size, while author covers keep their
|
|
42
|
+
* own dimensions (512x512, 768x768, 800x1200, 822x1200, 826x1169, 1024x1024 …).
|
|
43
|
+
*/
|
|
44
|
+
exports.PIXIV_GENERATED_COVER_WIDTH = 640;
|
|
45
|
+
exports.PIXIV_GENERATED_COVER_HEIGHT = 900;
|
|
46
|
+
/** Production-safe default: an unclassifiable cover is never shipped. */
|
|
47
|
+
exports.DEFAULT_NOVEL_COVER_POLICY = { unknownCover: 'skip' };
|
|
48
|
+
/**
|
|
49
|
+
* Classify fetched cover bytes. Returns `unknown` when the image header cannot
|
|
50
|
+
* be read (no decoder in this project, so only the header is inspected).
|
|
51
|
+
*/
|
|
52
|
+
function classifyNovelCover(cover) {
|
|
53
|
+
const dimensions = (0, imageDimensions_1.readImageDimensions)(cover);
|
|
54
|
+
if (!dimensions)
|
|
55
|
+
return 'unknown';
|
|
56
|
+
if (dimensions.width === exports.PIXIV_GENERATED_COVER_WIDTH &&
|
|
57
|
+
dimensions.height === exports.PIXIV_GENERATED_COVER_HEIGHT) {
|
|
58
|
+
return 'pixiv_generated';
|
|
59
|
+
}
|
|
60
|
+
return 'custom';
|
|
61
|
+
}
|
|
62
|
+
/** The delivery decision for one classified cover under a policy. */
|
|
63
|
+
function coverDeliveryDecision(policy, type) {
|
|
64
|
+
if (type === 'pixiv_generated')
|
|
65
|
+
return 'skip';
|
|
66
|
+
if (type === 'unknown')
|
|
67
|
+
return policy.unknownCover === 'keep' ? 'deliver' : 'skip';
|
|
68
|
+
return 'deliver';
|
|
69
|
+
}
|
|
70
|
+
//# sourceMappingURL=NovelCoverPolicy.js.map
|
|
@@ -6,6 +6,7 @@ const RankingService_1 = require("./RankingService");
|
|
|
6
6
|
const IllustrationDownloader_1 = require("./IllustrationDownloader");
|
|
7
7
|
const NovelDownloader_1 = require("./NovelDownloader");
|
|
8
8
|
const MaterializationPolicy_1 = require("../domain/media/MaterializationPolicy");
|
|
9
|
+
const NovelCoverPolicy_1 = require("../domain/media/NovelCoverPolicy");
|
|
9
10
|
const ProgressReporter_1 = require("./report/ProgressReporter");
|
|
10
11
|
const DownloadPlanner_1 = require("./plan/DownloadPlanner");
|
|
11
12
|
const DownloadExecutor_1 = require("./exec/DownloadExecutor");
|
|
@@ -129,7 +130,10 @@ class DownloadManager {
|
|
|
129
130
|
const materialization = config.download?.materializationPolicy
|
|
130
131
|
? { mode: config.download?.materializationPolicy }
|
|
131
132
|
: MaterializationPolicy_1.DEFAULT_MATERIALIZATION_POLICY;
|
|
132
|
-
|
|
133
|
+
const novelCoverPolicy = {
|
|
134
|
+
unknownCover: config.download?.novelCover?.unknown ?? NovelCoverPolicy_1.DEFAULT_NOVEL_COVER_POLICY.unknownCover,
|
|
135
|
+
};
|
|
136
|
+
this.novelDownloader = new NovelDownloader_1.NovelDownloader(client, database, fileService, database, undefined, materialization, novelCoverPolicy);
|
|
133
137
|
this.planner = new DownloadPlanner_1.DownloadPlanner(database, {
|
|
134
138
|
// `scope` is a single legacy target name, or the full fan-out array for a
|
|
135
139
|
// multi-platform target (then only works confirmed on EVERY platform count).
|
|
@@ -6,6 +6,7 @@ import { PixivNovel } from '@redtidev/pixiv-client';
|
|
|
6
6
|
import { DownloadedArtifact } from '../delivery/types';
|
|
7
7
|
import { type MaterializationPolicy } from '../domain/media/MaterializationPolicy';
|
|
8
8
|
import { type MediaMaterializer } from './materialization/MediaMaterializer';
|
|
9
|
+
import { type NovelCoverPolicy } from '../domain/media/NovelCoverPolicy';
|
|
9
10
|
import type { Database } from '../storage/Database';
|
|
10
11
|
export declare class NovelDownloader {
|
|
11
12
|
private readonly client;
|
|
@@ -14,7 +15,8 @@ export declare class NovelDownloader {
|
|
|
14
15
|
private readonly metadataDb?;
|
|
15
16
|
private readonly materializer;
|
|
16
17
|
private readonly materializationPolicy;
|
|
17
|
-
|
|
18
|
+
private readonly novelCoverPolicy;
|
|
19
|
+
constructor(client: IPixivClient, database: IDatabase, fileService: IFileService, metadataDb?: Database | undefined, materializer?: MediaMaterializer, materializationPolicy?: MaterializationPolicy, novelCoverPolicy?: NovelCoverPolicy);
|
|
18
20
|
download(novel: PixivNovel, tag: string, target: TargetConfig): Promise<DownloadedArtifact | undefined>;
|
|
19
21
|
/**
|
|
20
22
|
* Resolves the cover URL a novel should ship with (§novel-cover).
|
|
@@ -22,9 +24,18 @@ export declare class NovelDownloader {
|
|
|
22
24
|
* Pixiv's "author set no cover" case is invisible in the API: the design it
|
|
23
25
|
* renders (title typeset on a template) is served from the same CDN path with
|
|
24
26
|
* a unique hash as a real cover, so the URL cannot decide. The candidate cover
|
|
25
|
-
* is therefore fetched once and its header
|
|
26
|
-
* exactly 640x900.
|
|
27
|
-
*
|
|
27
|
+
* is therefore fetched once and classified from its header; Pixiv's design
|
|
28
|
+
* canvas is exactly 640x900.
|
|
29
|
+
*
|
|
30
|
+
* Policy (§media-asset-pipeline):
|
|
31
|
+
* - custom → deliver the cover
|
|
32
|
+
* - pixiv_generated → never deliver (a design cover is not content and must
|
|
33
|
+
* not become Telegram media)
|
|
34
|
+
* - unknown → policy decision, default 'skip' (safe mode), so a future
|
|
35
|
+
* Pixiv cover-format change surfaces as a loud
|
|
36
|
+
* `coverType=unknown` log instead of leaking silently
|
|
37
|
+
* A FAILED probe (network / auth / rate limit) is a separate case and always
|
|
38
|
+
* keeps the cover: a transient fetch error must never cost a real one.
|
|
28
39
|
*/
|
|
29
40
|
private resolveCoverUrl;
|
|
30
41
|
}
|
|
@@ -43,6 +43,7 @@ const Artifact_1 = require("../domain/media/Artifact");
|
|
|
43
43
|
const MediaMaterializer_1 = require("./materialization/MediaMaterializer");
|
|
44
44
|
const novelMarkers_1 = require("./novelMarkers");
|
|
45
45
|
const novelCover_1 = require("./novelCover");
|
|
46
|
+
const NovelCoverPolicy_1 = require("../domain/media/NovelCoverPolicy");
|
|
46
47
|
const zip_1 = require("../utils/zip");
|
|
47
48
|
const LANGUAGE_CACHE_TTL_MS = 7 * 24 * 60 * 60 * 1000;
|
|
48
49
|
class NovelDownloader {
|
|
@@ -52,13 +53,15 @@ class NovelDownloader {
|
|
|
52
53
|
metadataDb;
|
|
53
54
|
materializer;
|
|
54
55
|
materializationPolicy;
|
|
55
|
-
|
|
56
|
+
novelCoverPolicy;
|
|
57
|
+
constructor(client, database, fileService, metadataDb, materializer, materializationPolicy = MaterializationPolicy_1.DEFAULT_MATERIALIZATION_POLICY, novelCoverPolicy = NovelCoverPolicy_1.DEFAULT_NOVEL_COVER_POLICY) {
|
|
56
58
|
this.client = client;
|
|
57
59
|
this.database = database;
|
|
58
60
|
this.fileService = fileService;
|
|
59
61
|
this.metadataDb = metadataDb;
|
|
60
62
|
this.materializer = materializer ?? new MediaMaterializer_1.PixivMediaMaterializer(client, fileService);
|
|
61
63
|
this.materializationPolicy = materializationPolicy;
|
|
64
|
+
this.novelCoverPolicy = novelCoverPolicy;
|
|
62
65
|
}
|
|
63
66
|
async download(novel, tag, target) {
|
|
64
67
|
// Metadata cache: language filtering otherwise pulls FULL text per candidate
|
|
@@ -405,30 +408,59 @@ class NovelDownloader {
|
|
|
405
408
|
* Pixiv's "author set no cover" case is invisible in the API: the design it
|
|
406
409
|
* renders (title typeset on a template) is served from the same CDN path with
|
|
407
410
|
* a unique hash as a real cover, so the URL cannot decide. The candidate cover
|
|
408
|
-
* is therefore fetched once and its header
|
|
409
|
-
* exactly 640x900.
|
|
410
|
-
*
|
|
411
|
+
* is therefore fetched once and classified from its header; Pixiv's design
|
|
412
|
+
* canvas is exactly 640x900.
|
|
413
|
+
*
|
|
414
|
+
* Policy (§media-asset-pipeline):
|
|
415
|
+
* - custom → deliver the cover
|
|
416
|
+
* - pixiv_generated → never deliver (a design cover is not content and must
|
|
417
|
+
* not become Telegram media)
|
|
418
|
+
* - unknown → policy decision, default 'skip' (safe mode), so a future
|
|
419
|
+
* Pixiv cover-format change surfaces as a loud
|
|
420
|
+
* `coverType=unknown` log instead of leaking silently
|
|
421
|
+
* A FAILED probe (network / auth / rate limit) is a separate case and always
|
|
422
|
+
* keeps the cover: a transient fetch error must never cost a real one.
|
|
411
423
|
*/
|
|
412
424
|
async resolveCoverUrl(novelId, coverUrl) {
|
|
413
425
|
const normalized = (0, novelCover_1.normalizeNovelCoverUrl)(coverUrl);
|
|
414
426
|
if (!normalized)
|
|
415
427
|
return null;
|
|
428
|
+
let cover;
|
|
416
429
|
try {
|
|
417
|
-
|
|
418
|
-
if ((0, novelCover_1.isPixivDesignCoverImage)(cover)) {
|
|
419
|
-
logger_1.logger.info(`Novel ${novelId} cover is a Pixiv design cover (${novelCover_1.PIXIV_DESIGN_COVER_WIDTH}x${novelCover_1.PIXIV_DESIGN_COVER_HEIGHT}); delivering without a cover`, { novelId, coverUrl: normalized, designCover: true });
|
|
420
|
-
return null;
|
|
421
|
-
}
|
|
422
|
-
return normalized;
|
|
430
|
+
cover = await this.client.downloadImage(normalized);
|
|
423
431
|
}
|
|
424
432
|
catch (error) {
|
|
425
|
-
logger_1.logger.warn(`Novel ${novelId} cover probe failed; keeping the cover`, {
|
|
433
|
+
logger_1.logger.warn(`Novel ${novelId} cover probe failed; keeping the cover (coverType=probe_failed)`, {
|
|
426
434
|
novelId,
|
|
427
435
|
coverUrl: normalized,
|
|
436
|
+
coverType: 'probe_failed',
|
|
428
437
|
reason: error instanceof Error ? error.message : String(error),
|
|
429
438
|
});
|
|
430
439
|
return normalized;
|
|
431
440
|
}
|
|
441
|
+
const coverType = (0, NovelCoverPolicy_1.classifyNovelCover)(cover);
|
|
442
|
+
const decision = (0, NovelCoverPolicy_1.coverDeliveryDecision)(this.novelCoverPolicy, coverType);
|
|
443
|
+
const canvas = `${NovelCoverPolicy_1.PIXIV_GENERATED_COVER_WIDTH}x${NovelCoverPolicy_1.PIXIV_GENERATED_COVER_HEIGHT}`;
|
|
444
|
+
if (decision === 'skip') {
|
|
445
|
+
if (coverType === 'pixiv_generated') {
|
|
446
|
+
logger_1.logger.info(`Novel ${novelId} cover is a Pixiv generated design (${canvas}); delivering without a cover (coverType=${coverType})`, { novelId, coverUrl: normalized, coverType, canvas });
|
|
447
|
+
}
|
|
448
|
+
else {
|
|
449
|
+
logger_1.logger.warn(`Novel ${novelId} cover could not be classified; skipping it per novelCover.unknown=skip (coverType=${coverType})`, { novelId, coverUrl: normalized, coverType, policy: this.novelCoverPolicy.unknownCover });
|
|
450
|
+
}
|
|
451
|
+
return null;
|
|
452
|
+
}
|
|
453
|
+
if (coverType === 'unknown') {
|
|
454
|
+
logger_1.logger.warn(`Novel ${novelId} cover could not be classified; keeping it per novelCover.unknown=keep (coverType=${coverType})`, { novelId, coverUrl: normalized, coverType, policy: this.novelCoverPolicy.unknownCover });
|
|
455
|
+
}
|
|
456
|
+
else {
|
|
457
|
+
logger_1.logger.debug(`Novel ${novelId} cover classified (coverType=${coverType})`, {
|
|
458
|
+
novelId,
|
|
459
|
+
coverUrl: normalized,
|
|
460
|
+
coverType,
|
|
461
|
+
});
|
|
462
|
+
}
|
|
463
|
+
return normalized;
|
|
432
464
|
}
|
|
433
465
|
}
|
|
434
466
|
exports.NovelDownloader = NovelDownloader;
|
|
@@ -27,17 +27,16 @@
|
|
|
27
27
|
*/
|
|
28
28
|
import { MediaAsset } from '../domain/media/MediaAsset';
|
|
29
29
|
/**
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
* is delivered at exactly this size, while author covers keep their own
|
|
33
|
-
* dimensions (512x512, 768x768, 800x1200, 822x1200, 826x1169, 1024x1024 …).
|
|
30
|
+
* Historical names for the generated-cover canvas; the classification and the
|
|
31
|
+
* policy live in `src/domain/media/NovelCoverPolicy.ts` (§media-asset-pipeline).
|
|
34
32
|
*/
|
|
35
33
|
export declare const PIXIV_DESIGN_COVER_WIDTH = 640;
|
|
36
34
|
export declare const PIXIV_DESIGN_COVER_HEIGHT = 900;
|
|
37
35
|
/**
|
|
38
36
|
* True when the fetched cover bytes are one of Pixiv's generated designs.
|
|
39
|
-
* Unknown formats / unreadable payloads return false
|
|
40
|
-
*
|
|
37
|
+
* Unknown formats / unreadable payloads return false — this predicate is the
|
|
38
|
+
* raw classification only; whether an unclassifiable cover is delivered is the
|
|
39
|
+
* caller's policy (`coverDeliveryDecision`, default: skipped).
|
|
41
40
|
*/
|
|
42
41
|
export declare function isPixivDesignCoverImage(cover: ArrayBuffer | Uint8Array | null | undefined): boolean;
|
|
43
42
|
export declare function normalizeNovelCoverUrl(coverUrl?: string | null): string | null;
|
|
@@ -32,27 +32,24 @@ exports.novelCoverAsset = novelCoverAsset;
|
|
|
32
32
|
* positional guessing.
|
|
33
33
|
*/
|
|
34
34
|
const MediaAsset_1 = require("../domain/media/MediaAsset");
|
|
35
|
-
const
|
|
35
|
+
const NovelCoverPolicy_1 = require("../domain/media/NovelCoverPolicy");
|
|
36
36
|
const DEFAULT_COVER_PATTERN = /novel-cover-(master-)?default/i;
|
|
37
37
|
/** Strip the `/c/<spec>/` resizer segment to recover the original-size URL. */
|
|
38
38
|
const RESIZED_COVER_PATTERN = /\/c\/[^/]+\/(novel-cover-master\/)/;
|
|
39
39
|
/**
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
* is delivered at exactly this size, while author covers keep their own
|
|
43
|
-
* dimensions (512x512, 768x768, 800x1200, 822x1200, 826x1169, 1024x1024 …).
|
|
40
|
+
* Historical names for the generated-cover canvas; the classification and the
|
|
41
|
+
* policy live in `src/domain/media/NovelCoverPolicy.ts` (§media-asset-pipeline).
|
|
44
42
|
*/
|
|
45
|
-
exports.PIXIV_DESIGN_COVER_WIDTH =
|
|
46
|
-
exports.PIXIV_DESIGN_COVER_HEIGHT =
|
|
43
|
+
exports.PIXIV_DESIGN_COVER_WIDTH = NovelCoverPolicy_1.PIXIV_GENERATED_COVER_WIDTH;
|
|
44
|
+
exports.PIXIV_DESIGN_COVER_HEIGHT = NovelCoverPolicy_1.PIXIV_GENERATED_COVER_HEIGHT;
|
|
47
45
|
/**
|
|
48
46
|
* True when the fetched cover bytes are one of Pixiv's generated designs.
|
|
49
|
-
* Unknown formats / unreadable payloads return false
|
|
50
|
-
*
|
|
47
|
+
* Unknown formats / unreadable payloads return false — this predicate is the
|
|
48
|
+
* raw classification only; whether an unclassifiable cover is delivered is the
|
|
49
|
+
* caller's policy (`coverDeliveryDecision`, default: skipped).
|
|
51
50
|
*/
|
|
52
51
|
function isPixivDesignCoverImage(cover) {
|
|
53
|
-
|
|
54
|
-
return (dimensions?.width === exports.PIXIV_DESIGN_COVER_WIDTH &&
|
|
55
|
-
dimensions?.height === exports.PIXIV_DESIGN_COVER_HEIGHT);
|
|
52
|
+
return (0, NovelCoverPolicy_1.classifyNovelCover)(cover) === 'pixiv_generated';
|
|
56
53
|
}
|
|
57
54
|
function normalizeNovelCoverUrl(coverUrl) {
|
|
58
55
|
const url = typeof coverUrl === 'string' ? coverUrl.trim() : '';
|
package/dist/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { TargetConfig } from '../config';
|
|
2
2
|
import type { TopicResolver } from './TopicResolver';
|
|
3
|
-
import type { TopicCandidate, TopicClient, TopicCollectOptions, TopicContentType, TopicDiscoveryOptions, WorkLike } from './types';
|
|
3
|
+
import type { RelatedTagMode, ResolvedTag, TopicCandidate, TopicClient, TopicCollectOptions, TopicContentType, TopicDiscoveryOptions, TopicRelationsOptions, WorkLike } from './types';
|
|
4
4
|
export interface TopicSelection {
|
|
5
5
|
candidates: TopicCandidate[];
|
|
6
6
|
selected: TopicCandidate[];
|
|
@@ -18,6 +18,26 @@ export interface TopicSelection {
|
|
|
18
18
|
/** Works seen more than once across the topic tag space (recorded, dropped). */
|
|
19
19
|
duplicateRemovedCount: number;
|
|
20
20
|
}
|
|
21
|
+
/**
|
|
22
|
+
* The tag the operator asked for, even when `tagRelations` filters it out.
|
|
23
|
+
*
|
|
24
|
+
* `deny` may drop the seed, but a space filtered down to nothing must still be
|
|
25
|
+
* usable: the pipeline then falls back to the seed tag alone, exactly like the
|
|
26
|
+
* resolver's `seedOnly()` degradation. `weight` defaults to the tag's `score`
|
|
27
|
+
* for spaces persisted before provenance existed.
|
|
28
|
+
*/
|
|
29
|
+
export declare function seedFallbackTag(seedTag: ResolvedTag): ResolvedTag;
|
|
30
|
+
/**
|
|
31
|
+
* Applies `topicDiscovery.tagRelations` to a resolved space (§tag-provenance).
|
|
32
|
+
*
|
|
33
|
+
* Deny beats allow, and deny is the only thing that can drop the seed tag. A
|
|
34
|
+
* tag whose provenance is unknown (a space persisted before `source` existed, or
|
|
35
|
+
* a hand-written space) is treated as `cooccurrence` so that `allowSources` can
|
|
36
|
+
* still narrow it while the default (all sources) keeps walking everything.
|
|
37
|
+
*/
|
|
38
|
+
export declare function selectWalkedTags(tags: readonly ResolvedTag[], seedKey: string, relations?: TopicRelationsOptions): ResolvedTag[];
|
|
39
|
+
/** Which of the walked tags a `relatedTags` mode will actually search today. */
|
|
40
|
+
export declare function recallChannels(walked: readonly ResolvedTag[], seedKey: string, relatedMode: RelatedTagMode): ResolvedTag[];
|
|
21
41
|
/**
|
|
22
42
|
* Resolves a topic to a tag space, collects that day's works across the tags,
|
|
23
43
|
* filters by lightweight metadata relevance and ranks by local popularity, then
|
|
@@ -63,6 +83,16 @@ export declare class TopicPipeline {
|
|
|
63
83
|
* Lightweight metadata relevance. Tags dominate (Pixiv's own taxonomy);
|
|
64
84
|
* title/caption add smaller boosts. The seed tag is strong evidence.
|
|
65
85
|
* No text model — case/symbol-insensitive substring matching only.
|
|
86
|
+
*
|
|
87
|
+
* `matchTranslatedNames` (default false) additionally counts a work's
|
|
88
|
+
* `translated_name` as a hit for the resolved tag it translates to, so a
|
|
89
|
+
* topic written in one language can still match works tagged in another. The
|
|
90
|
+
* translated name is matched against the resolved space — never added to
|
|
91
|
+
* `candidate.tags` — so the work is not reported as carrying a tag it lacks.
|
|
92
|
+
*
|
|
93
|
+
* A resolved tag counts at most ONCE per work, whether it matched through the
|
|
94
|
+
* tag name or through its translation (and a work that repeats a tag, as Pixiv
|
|
95
|
+
* payloads do for a tag plus its translation, does not double its weight).
|
|
66
96
|
*/
|
|
67
97
|
private metadataScore;
|
|
68
98
|
/**
|
|
@@ -72,12 +102,13 @@ export declare class TopicPipeline {
|
|
|
72
102
|
* alone. A work with a higher metadata score does NOT outrank a more popular
|
|
73
103
|
* accepted work.
|
|
74
104
|
*
|
|
75
|
-
* `seedKey` adds a single tier in front of that popularity order and is
|
|
76
|
-
* passed by the seed-first recall modes (§topic-recall)
|
|
77
|
-
*
|
|
78
|
-
*
|
|
79
|
-
*
|
|
80
|
-
* is
|
|
105
|
+
* `seedKey` adds a single tier in front of that popularity order and is
|
|
106
|
+
* passed by the seed-first recall modes (§topic-recall) and by
|
|
107
|
+
* `topicDiscovery.seedTier: 'on'` (§tag-provenance): a work that actually
|
|
108
|
+
* carries the topic tag outranks a related-only work, however popular the
|
|
109
|
+
* latter is, and popularity decides within each tier. With `seedTier: 'off'`
|
|
110
|
+
* and the default `relatedTags: 'always'` no `seedKey` is passed, so the
|
|
111
|
+
* documented popularity-only ranking is unchanged.
|
|
81
112
|
*/
|
|
82
113
|
private popCompare;
|
|
83
114
|
private rankCompare;
|