pixivflow 2.38.0 → 2.40.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/delivery/TelePressRichNovel.d.ts +7 -0
- package/dist/delivery/TelePressRichNovel.js +74 -15
- package/dist/delivery/types.d.ts +6 -0
- package/dist/domain/media/Artifact.d.ts +32 -0
- package/dist/domain/media/Artifact.js +24 -0
- package/dist/domain/media/MaterializationPolicy.d.ts +13 -0
- package/dist/domain/media/MaterializationPolicy.js +11 -0
- package/dist/domain/media/MediaAsset.d.ts +49 -0
- package/dist/domain/media/MediaAsset.js +31 -0
- package/dist/domain/media/Work.d.ts +27 -0
- package/dist/domain/media/Work.js +12 -0
- package/dist/download/NovelDownloader.d.ts +5 -1
- package/dist/download/NovelDownloader.js +97 -28
- package/dist/download/materialization/MediaMaterializer.d.ts +24 -0
- package/dist/download/materialization/MediaMaterializer.js +44 -0
- package/dist/download/novelMarkers.d.ts +14 -0
- package/dist/download/novelMarkers.js +48 -0
- package/dist/package.json +1 -1
- package/dist/version.js +1 -1
- package/dist/webui/package.json +1 -1
- package/package.json +1 -1
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { DownloadedArtifact } from './types';
|
|
2
|
+
import { type MediaAsset } from '../domain/media/MediaAsset';
|
|
2
3
|
export interface RichNovelPreviewResult {
|
|
3
4
|
/** Telegraph "read online" URL for the published page. */
|
|
4
5
|
url: string;
|
|
@@ -25,11 +26,17 @@ export interface RichNovelManifestEntry {
|
|
|
25
26
|
local: string;
|
|
26
27
|
/** Original Pixiv CDN source URL (``https://i.pximg.net/...``). */
|
|
27
28
|
source: string;
|
|
29
|
+
/** Canonical MediaAsset id (new contract, optional for backward compat). */
|
|
30
|
+
assetId?: string;
|
|
31
|
+
/** Canonical source URL (new contract; same as ``source`` today). */
|
|
32
|
+
sourceUrl?: string;
|
|
28
33
|
}
|
|
29
34
|
export interface RichNovelSources {
|
|
30
35
|
txtPath: string;
|
|
31
36
|
mdPath: string;
|
|
32
37
|
imagePaths: string[];
|
|
38
|
+
/** Canonical media references derived from the novel metadata file. */
|
|
39
|
+
mediaAssets: MediaAsset[];
|
|
33
40
|
/** Optional Pixiv CDN source map derived from the novel metadata file. */
|
|
34
41
|
manifest: RichNovelManifestEntry[];
|
|
35
42
|
}
|
|
@@ -39,6 +39,8 @@ exports.publishRichNovelPreview = publishRichNovelPreview;
|
|
|
39
39
|
const fs = __importStar(require("node:fs"));
|
|
40
40
|
const path = __importStar(require("node:path"));
|
|
41
41
|
const node_crypto_1 = require("node:crypto");
|
|
42
|
+
const MediaAsset_1 = require("../domain/media/MediaAsset");
|
|
43
|
+
const novelMarkers_1 = require("../download/novelMarkers");
|
|
42
44
|
const logger_1 = require("../logger");
|
|
43
45
|
const redact_1 = require("../utils/redact");
|
|
44
46
|
/**
|
|
@@ -55,29 +57,84 @@ function interpolateEnv(value) {
|
|
|
55
57
|
return resolved;
|
|
56
58
|
});
|
|
57
59
|
}
|
|
58
|
-
|
|
59
|
-
|
|
60
|
+
function isDownloadedPixivAsset(asset) {
|
|
61
|
+
return Boolean(asset && asset.status === 'downloaded' && asset.url && asset.localPath);
|
|
62
|
+
}
|
|
63
|
+
function isPendingPixivAsset(asset) {
|
|
64
|
+
return Boolean(asset && asset.status === 'pending' && asset.url && asset.sourceId && !asset.localPath);
|
|
65
|
+
}
|
|
66
|
+
/** Build a canonical MediaAsset without requiring a local file. */
|
|
67
|
+
function mediaAssetFrom(asset, workId, localPath) {
|
|
68
|
+
const kind = asset.kind === 'uploadedimage' || asset.kind === 'pixivimage'
|
|
69
|
+
? asset.kind
|
|
70
|
+
: undefined;
|
|
71
|
+
const source = String(asset.url);
|
|
72
|
+
if (!source || !kind)
|
|
73
|
+
return undefined;
|
|
74
|
+
return (0, MediaAsset_1.buildMediaAsset)({
|
|
75
|
+
workId,
|
|
76
|
+
kind,
|
|
77
|
+
sourceId: asset.sourceId ? String(asset.sourceId) : undefined,
|
|
78
|
+
marker: asset.marker ? String(asset.marker) : undefined,
|
|
79
|
+
sourceUrl: source,
|
|
80
|
+
artifactId: localPath,
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Read the novel metadata sidecar into canonical `MediaAsset` values, plus the
|
|
85
|
+
* legacy `{local, source}` manifest projection (local is only a render hint for
|
|
86
|
+
* the markdown refs; source is the media fact).
|
|
87
|
+
*/
|
|
88
|
+
function readNovelMediaAssets(artifact) {
|
|
60
89
|
const metadataFile = (artifact.cleanupFiles ?? []).find((f) => /\.json$/i.test(f));
|
|
61
|
-
if (!metadataFile || !fs.existsSync(metadataFile))
|
|
62
|
-
return [];
|
|
90
|
+
if (!metadataFile || !fs.existsSync(metadataFile)) {
|
|
91
|
+
return { mediaAssets: [], manifest: [] };
|
|
92
|
+
}
|
|
63
93
|
try {
|
|
64
94
|
const meta = JSON.parse(fs.readFileSync(metadataFile, 'utf8'));
|
|
65
95
|
if (!Array.isArray(meta.assets))
|
|
66
|
-
return [];
|
|
67
|
-
const
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
96
|
+
return { mediaAssets: [], manifest: [] };
|
|
97
|
+
const workId = meta.pixiv_id ? String(meta.pixiv_id) : artifact.pixivId;
|
|
98
|
+
const mediaAssets = [];
|
|
99
|
+
const manifest = [];
|
|
100
|
+
const downloaded = meta.assets.filter(isDownloadedPixivAsset);
|
|
101
|
+
for (const asset of downloaded) {
|
|
71
102
|
const source = String(asset.url);
|
|
72
103
|
const localPath = String(asset.localPath);
|
|
73
|
-
|
|
104
|
+
const media = mediaAssetFrom(asset, workId, localPath);
|
|
105
|
+
if (!media)
|
|
74
106
|
continue;
|
|
75
|
-
|
|
107
|
+
mediaAssets.push(media);
|
|
108
|
+
manifest.push({
|
|
109
|
+
local: `images/${path.basename(localPath)}`,
|
|
110
|
+
source,
|
|
111
|
+
assetId: mediaAssets[mediaAssets.length - 1]?.id,
|
|
112
|
+
sourceUrl: source,
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
// On-demand previews: no local downloads for this work, but the metadata
|
|
116
|
+
// carries resolvable Pixiv CDN sources. Pass those through as pending
|
|
117
|
+
// references (no artifactId) so TelePress can proxy them without files.
|
|
118
|
+
if (mediaAssets.length === 0) {
|
|
119
|
+
for (const asset of meta.assets.filter(isPendingPixivAsset)) {
|
|
120
|
+
const source = String(asset.url);
|
|
121
|
+
const refName = (0, novelMarkers_1.novelReferenceFileName)(source, String(asset.sourceId));
|
|
122
|
+
const media = mediaAssetFrom(asset, workId);
|
|
123
|
+
if (!media)
|
|
124
|
+
continue;
|
|
125
|
+
mediaAssets.push(media);
|
|
126
|
+
manifest.push({
|
|
127
|
+
local: refName,
|
|
128
|
+
source,
|
|
129
|
+
assetId: mediaAssets[mediaAssets.length - 1]?.id,
|
|
130
|
+
sourceUrl: source,
|
|
131
|
+
});
|
|
132
|
+
}
|
|
76
133
|
}
|
|
77
|
-
return
|
|
134
|
+
return { mediaAssets, manifest };
|
|
78
135
|
}
|
|
79
136
|
catch {
|
|
80
|
-
return [];
|
|
137
|
+
return { mediaAssets: [], manifest: [] };
|
|
81
138
|
}
|
|
82
139
|
}
|
|
83
140
|
function findRichNovelSources(artifact) {
|
|
@@ -98,11 +155,13 @@ function findRichNovelSources(artifact) {
|
|
|
98
155
|
.map((name) => path.join(imagesDir, name))
|
|
99
156
|
.sort((a, b) => a.localeCompare(b, undefined, { numeric: true }));
|
|
100
157
|
}
|
|
158
|
+
const { mediaAssets, manifest } = readNovelMediaAssets(artifact);
|
|
101
159
|
return {
|
|
102
160
|
txtPath,
|
|
103
161
|
mdPath,
|
|
104
162
|
imagePaths,
|
|
105
|
-
|
|
163
|
+
mediaAssets,
|
|
164
|
+
manifest,
|
|
106
165
|
};
|
|
107
166
|
}
|
|
108
167
|
/**
|
|
@@ -116,7 +175,7 @@ async function publishRichNovelPreview(artifact, options) {
|
|
|
116
175
|
const sources = findRichNovelSources(artifact);
|
|
117
176
|
if (!sources)
|
|
118
177
|
return { url: '', retryable: false, operatorHint: 'no_rich_novel_assets' };
|
|
119
|
-
if (sources.imagePaths.length === 0) {
|
|
178
|
+
if (sources.imagePaths.length === 0 && sources.manifest.length === 0) {
|
|
120
179
|
return { url: '', retryable: false, operatorHint: 'no_rich_novel_assets' };
|
|
121
180
|
}
|
|
122
181
|
const mdName = path.basename(sources.mdPath);
|
package/dist/delivery/types.d.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { DeliveryFieldValue } from '../config';
|
|
2
|
+
import type { MediaAsset } from '../domain/media/MediaAsset';
|
|
3
|
+
import type { Artifact } from '../domain/media/Artifact';
|
|
2
4
|
export type DeliveryItemType = 'illustration' | 'novel';
|
|
3
5
|
/** Files produced for one Pixiv work and needed by a delivery provider. */
|
|
4
6
|
export interface DownloadedArtifact {
|
|
@@ -11,6 +13,10 @@ export interface DownloadedArtifact {
|
|
|
11
13
|
files: string[];
|
|
12
14
|
/** Optional per-file lightweight preview sources, aligned with ``files``. */
|
|
13
15
|
previewFiles?: string[];
|
|
16
|
+
/** Canonical media facts for the work (remote source, stable id). */
|
|
17
|
+
mediaAssets?: MediaAsset[];
|
|
18
|
+
/** Canonical materialized file facts backing ``files``. */
|
|
19
|
+
artifacts?: Artifact[];
|
|
14
20
|
/** Local sidecars deleted with cache files after successful delivery. */
|
|
15
21
|
cleanupFiles?: string[];
|
|
16
22
|
/** R-18 work (x_restrict > 0): delivery templates may open Telegram spoiler. */
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A materialized file produced for a work.
|
|
3
|
+
*
|
|
4
|
+
* `Artifact` is the file fact (which file, which variant), separate from
|
|
5
|
+
* `MediaAsset` (the media fact). Legacy `DownloadedArtifact.files[]` remains a
|
|
6
|
+
* compatibility projection until all consumers migrate.
|
|
7
|
+
*/
|
|
8
|
+
export type ArtifactVariant = 'original' | 'text' | 'markdown' | 'zip' | 'metadata' | 'delivery';
|
|
9
|
+
export interface Artifact {
|
|
10
|
+
/** Stable deterministic id, derived from the work + variant, not delivery state. */
|
|
11
|
+
id: string;
|
|
12
|
+
/** When this file is a materialized medium, the source MediaAsset id. */
|
|
13
|
+
sourceAssetId?: string;
|
|
14
|
+
workId: string;
|
|
15
|
+
variant: ArtifactVariant;
|
|
16
|
+
path: string;
|
|
17
|
+
mimeType?: string;
|
|
18
|
+
size?: number;
|
|
19
|
+
checksum?: string;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Deterministic artifact identity: `pixiv:<workId>:<variant>:<basename>`.
|
|
23
|
+
* The basename may be empty for generated variants; callers should pass a sane
|
|
24
|
+
* label so the id stays unique within a work.
|
|
25
|
+
*/
|
|
26
|
+
export declare function artifactId(workId: string, variant: ArtifactVariant, label: string): string;
|
|
27
|
+
/**
|
|
28
|
+
* Legacy projection: old consumers can keep reading `files[]`, which the new
|
|
29
|
+
* canonical list of artifacts backs. Only file-backed variants are projected.
|
|
30
|
+
*/
|
|
31
|
+
export declare function projectLegacyFiles(artifacts: Artifact[]): string[];
|
|
32
|
+
//# sourceMappingURL=Artifact.d.ts.map
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.artifactId = artifactId;
|
|
4
|
+
exports.projectLegacyFiles = projectLegacyFiles;
|
|
5
|
+
/**
|
|
6
|
+
* Deterministic artifact identity: `pixiv:<workId>:<variant>:<basename>`.
|
|
7
|
+
* The basename may be empty for generated variants; callers should pass a sane
|
|
8
|
+
* label so the id stays unique within a work.
|
|
9
|
+
*/
|
|
10
|
+
function artifactId(workId, variant, label) {
|
|
11
|
+
const cleanWork = String(workId).trim();
|
|
12
|
+
const cleanLabel = String(label).trim();
|
|
13
|
+
return `pixiv:${cleanWork}:${variant}${cleanLabel ? `:${cleanLabel}` : ''}`;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Legacy projection: old consumers can keep reading `files[]`, which the new
|
|
17
|
+
* canonical list of artifacts backs. Only file-backed variants are projected.
|
|
18
|
+
*/
|
|
19
|
+
function projectLegacyFiles(artifacts) {
|
|
20
|
+
return artifacts
|
|
21
|
+
.filter((a) => Boolean(a && a.path))
|
|
22
|
+
.map((a) => a.path);
|
|
23
|
+
}
|
|
24
|
+
//# sourceMappingURL=Artifact.js.map
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Materialization policy: whether a resolved MediaAsset becomes a local file
|
|
3
|
+
* eagerly or is deferred for a consumer.
|
|
4
|
+
*/
|
|
5
|
+
export type MaterializationMode = 'eager' | 'on-demand';
|
|
6
|
+
export interface MaterializationPolicy {
|
|
7
|
+
mode: MaterializationMode;
|
|
8
|
+
}
|
|
9
|
+
/** Production-safe default: eager (current behavior unchanged). */
|
|
10
|
+
export declare const DEFAULT_MATERIALIZATION_POLICY: MaterializationPolicy;
|
|
11
|
+
/** True when the policy says a resolved medium should be materialized now. */
|
|
12
|
+
export declare function shouldMaterialize(policy: MaterializationPolicy): boolean;
|
|
13
|
+
//# sourceMappingURL=MaterializationPolicy.d.ts.map
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.DEFAULT_MATERIALIZATION_POLICY = void 0;
|
|
4
|
+
exports.shouldMaterialize = shouldMaterialize;
|
|
5
|
+
/** Production-safe default: eager (current behavior unchanged). */
|
|
6
|
+
exports.DEFAULT_MATERIALIZATION_POLICY = { mode: 'eager' };
|
|
7
|
+
/** True when the policy says a resolved medium should be materialized now. */
|
|
8
|
+
function shouldMaterialize(policy) {
|
|
9
|
+
return policy.mode === 'eager';
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=MaterializationPolicy.js.map
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Canonical media reference for a download work.
|
|
3
|
+
*
|
|
4
|
+
* `MediaAsset` is the media fact (remote source + stable identity). It is NOT a
|
|
5
|
+
* local file: consumers decide whether / when to materialize it. The legacy
|
|
6
|
+
* `DownloadedArtifact.files[]` compatibility projection stays in place until all
|
|
7
|
+
* consumers migrate.
|
|
8
|
+
*/
|
|
9
|
+
export type PixivMediaKind = 'uploadedimage' | 'pixivimage' | 'illust';
|
|
10
|
+
export interface MediaSourceRef {
|
|
11
|
+
/** Pixiv work id, e.g. the novel/illust id. */
|
|
12
|
+
workId: string;
|
|
13
|
+
/** Pixiv-side image identifier (illust page / uploaded image id). */
|
|
14
|
+
sourceId?: string;
|
|
15
|
+
/** Original in-text marker, when known. */
|
|
16
|
+
marker?: string;
|
|
17
|
+
/** Pixiv-specific media subtype (uploadedimage vs pixivimage vs illust). */
|
|
18
|
+
pixivKind?: PixivMediaKind;
|
|
19
|
+
}
|
|
20
|
+
export interface MediaAsset {
|
|
21
|
+
/** Stable deterministic id, independent of any consumer/delivery system. */
|
|
22
|
+
id: string;
|
|
23
|
+
source: 'pixiv';
|
|
24
|
+
kind: 'image';
|
|
25
|
+
sourceUrl: string;
|
|
26
|
+
mimeType?: string;
|
|
27
|
+
width?: number;
|
|
28
|
+
height?: number;
|
|
29
|
+
page?: number;
|
|
30
|
+
/** Optional materialized file reference; never required. */
|
|
31
|
+
artifactId?: string;
|
|
32
|
+
sourceRef?: MediaSourceRef;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Deterministic stable identity: `pixiv:<workId>:<pixivKind>[:<sourceId>]`.
|
|
36
|
+
* Never carries Telegram/thelegraph/catbox ids.
|
|
37
|
+
*/
|
|
38
|
+
export declare function mediaAssetId(workId: string, pixivKind: PixivMediaKind, sourceId?: string): string;
|
|
39
|
+
export interface MediaAssetInput {
|
|
40
|
+
workId: string;
|
|
41
|
+
kind: PixivMediaKind;
|
|
42
|
+
sourceId?: string;
|
|
43
|
+
marker?: string;
|
|
44
|
+
sourceUrl: string;
|
|
45
|
+
artifactId?: string;
|
|
46
|
+
}
|
|
47
|
+
/** Build a canonical MediaAsset from a resolved Pixiv media reference. */
|
|
48
|
+
export declare function buildMediaAsset(input: MediaAssetInput): MediaAsset;
|
|
49
|
+
//# sourceMappingURL=MediaAsset.d.ts.map
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.mediaAssetId = mediaAssetId;
|
|
4
|
+
exports.buildMediaAsset = buildMediaAsset;
|
|
5
|
+
/**
|
|
6
|
+
* Deterministic stable identity: `pixiv:<workId>:<pixivKind>[:<sourceId>]`.
|
|
7
|
+
* Never carries Telegram/thelegraph/catbox ids.
|
|
8
|
+
*/
|
|
9
|
+
function mediaAssetId(workId, pixivKind, sourceId) {
|
|
10
|
+
const cleanWork = String(workId).trim();
|
|
11
|
+
const cleanKind = String(pixivKind).trim();
|
|
12
|
+
const cleanSource = sourceId ? String(sourceId).trim() : '';
|
|
13
|
+
return `pixiv:${cleanWork}:${cleanKind}${cleanSource ? `:${cleanSource}` : ''}`;
|
|
14
|
+
}
|
|
15
|
+
/** Build a canonical MediaAsset from a resolved Pixiv media reference. */
|
|
16
|
+
function buildMediaAsset(input) {
|
|
17
|
+
return {
|
|
18
|
+
id: mediaAssetId(input.workId, input.kind, input.sourceId),
|
|
19
|
+
source: 'pixiv',
|
|
20
|
+
kind: 'image',
|
|
21
|
+
sourceUrl: input.sourceUrl,
|
|
22
|
+
artifactId: input.artifactId,
|
|
23
|
+
sourceRef: {
|
|
24
|
+
workId: input.workId,
|
|
25
|
+
sourceId: input.sourceId,
|
|
26
|
+
marker: input.marker,
|
|
27
|
+
pixivKind: input.kind,
|
|
28
|
+
},
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
//# sourceMappingURL=MediaAsset.js.map
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { type MediaAsset, type PixivMediaKind } from './MediaAsset';
|
|
2
|
+
/** Minimal domain work descriptor; does not carry scheduler/delivery state. */
|
|
3
|
+
export interface Work {
|
|
4
|
+
id: string;
|
|
5
|
+
type: 'novel' | 'illustration';
|
|
6
|
+
title?: string;
|
|
7
|
+
sourceUrl?: string;
|
|
8
|
+
tags?: string[];
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Result of resolving a work WITHOUT materializing files: the work descriptor
|
|
12
|
+
* plus the canonical media facts consumers may choose to materialize.
|
|
13
|
+
*/
|
|
14
|
+
export interface ResolvedWork {
|
|
15
|
+
work: Work;
|
|
16
|
+
mediaAssets: MediaAsset[];
|
|
17
|
+
}
|
|
18
|
+
export interface WorkMediaInput {
|
|
19
|
+
workId: string;
|
|
20
|
+
kind: PixivMediaKind;
|
|
21
|
+
sourceId?: string;
|
|
22
|
+
marker?: string;
|
|
23
|
+
sourceUrl: string;
|
|
24
|
+
}
|
|
25
|
+
/** Build a ResolvedWork from already-fetched Pixiv detail + inline media refs. */
|
|
26
|
+
export declare function toResolvedWork(work: Work, mediaInputs: WorkMediaInput[]): ResolvedWork;
|
|
27
|
+
//# sourceMappingURL=Work.d.ts.map
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.toResolvedWork = toResolvedWork;
|
|
4
|
+
const MediaAsset_1 = require("./MediaAsset");
|
|
5
|
+
/** Build a ResolvedWork from already-fetched Pixiv detail + inline media refs. */
|
|
6
|
+
function toResolvedWork(work, mediaInputs) {
|
|
7
|
+
return {
|
|
8
|
+
work,
|
|
9
|
+
mediaAssets: mediaInputs.map((m) => (0, MediaAsset_1.buildMediaAsset)(m)),
|
|
10
|
+
};
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=Work.js.map
|
|
@@ -4,13 +4,17 @@ import { IDatabase } from '../interfaces/IDatabase';
|
|
|
4
4
|
import { IFileService } from '../interfaces/IFileService';
|
|
5
5
|
import { PixivNovel } from '@redtidev/pixiv-client';
|
|
6
6
|
import { DownloadedArtifact } from '../delivery/types';
|
|
7
|
+
import { type MaterializationPolicy } from '../domain/media/MaterializationPolicy';
|
|
8
|
+
import { type MediaMaterializer } from './materialization/MediaMaterializer';
|
|
7
9
|
import type { Database } from '../storage/Database';
|
|
8
10
|
export declare class NovelDownloader {
|
|
9
11
|
private readonly client;
|
|
10
12
|
private readonly database;
|
|
11
13
|
private readonly fileService;
|
|
12
14
|
private readonly metadataDb?;
|
|
13
|
-
|
|
15
|
+
private readonly materializer;
|
|
16
|
+
private readonly materializationPolicy;
|
|
17
|
+
constructor(client: IPixivClient, database: IDatabase, fileService: IFileService, metadataDb?: Database | undefined, materializer?: MediaMaterializer, materializationPolicy?: MaterializationPolicy);
|
|
14
18
|
download(novel: PixivNovel, tag: string, target: TargetConfig): Promise<DownloadedArtifact | undefined>;
|
|
15
19
|
}
|
|
16
20
|
//# sourceMappingURL=NovelDownloader.d.ts.map
|
|
@@ -37,6 +37,10 @@ exports.NovelDownloader = void 0;
|
|
|
37
37
|
const logger_1 = require("../logger");
|
|
38
38
|
const node_path_1 = require("node:path");
|
|
39
39
|
const language_detection_1 = require("../utils/language-detection");
|
|
40
|
+
const Work_1 = require("../domain/media/Work");
|
|
41
|
+
const MaterializationPolicy_1 = require("../domain/media/MaterializationPolicy");
|
|
42
|
+
const Artifact_1 = require("../domain/media/Artifact");
|
|
43
|
+
const MediaMaterializer_1 = require("./materialization/MediaMaterializer");
|
|
40
44
|
const novelMarkers_1 = require("./novelMarkers");
|
|
41
45
|
const zip_1 = require("../utils/zip");
|
|
42
46
|
const LANGUAGE_CACHE_TTL_MS = 7 * 24 * 60 * 60 * 1000;
|
|
@@ -45,11 +49,15 @@ class NovelDownloader {
|
|
|
45
49
|
database;
|
|
46
50
|
fileService;
|
|
47
51
|
metadataDb;
|
|
48
|
-
|
|
52
|
+
materializer;
|
|
53
|
+
materializationPolicy;
|
|
54
|
+
constructor(client, database, fileService, metadataDb, materializer, materializationPolicy = MaterializationPolicy_1.DEFAULT_MATERIALIZATION_POLICY) {
|
|
49
55
|
this.client = client;
|
|
50
56
|
this.database = database;
|
|
51
57
|
this.fileService = fileService;
|
|
52
58
|
this.metadataDb = metadataDb;
|
|
59
|
+
this.materializer = materializer ?? new MediaMaterializer_1.PixivMediaMaterializer(client, fileService);
|
|
60
|
+
this.materializationPolicy = materializationPolicy;
|
|
53
61
|
}
|
|
54
62
|
async download(novel, tag, target) {
|
|
55
63
|
// Metadata cache: language filtering otherwise pulls FULL text per candidate
|
|
@@ -163,22 +171,52 @@ class NovelDownloader {
|
|
|
163
171
|
? []
|
|
164
172
|
: (0, novelMarkers_1.extractNovelAssets)(text, textResponse);
|
|
165
173
|
const hadImages = assets.length > 0;
|
|
166
|
-
|
|
167
|
-
|
|
174
|
+
// Step 6: resolve first (work + media facts), materialize second.
|
|
175
|
+
const resolved = (0, Work_1.toResolvedWork)({
|
|
176
|
+
id: String(detail.id),
|
|
177
|
+
type: 'novel',
|
|
178
|
+
title: detail.title,
|
|
179
|
+
sourceUrl: `https://www.pixiv.net/novel/show.php?id=${detail.id}`,
|
|
180
|
+
tags: tags.map((item) => item.name).filter(Boolean),
|
|
181
|
+
}, assets
|
|
182
|
+
.filter((a) => Boolean(a.url))
|
|
183
|
+
.map((a) => ({
|
|
184
|
+
workId: String(detail.id),
|
|
185
|
+
kind: a.kind,
|
|
186
|
+
sourceId: a.sourceId,
|
|
187
|
+
marker: a.marker,
|
|
188
|
+
sourceUrl: a.url,
|
|
189
|
+
})));
|
|
190
|
+
if (resolved.mediaAssets.length) {
|
|
168
191
|
const imagesDir = (0, node_path_1.join)((0, node_path_1.dirname)(filePath), 'images');
|
|
169
|
-
for (const
|
|
192
|
+
for (const mediaAsset of resolved.mediaAssets) {
|
|
193
|
+
if (!(0, MaterializationPolicy_1.shouldMaterialize)(this.materializationPolicy)) {
|
|
194
|
+
continue; // on-demand: keep the media reference, defer local file creation
|
|
195
|
+
}
|
|
170
196
|
try {
|
|
171
|
-
const
|
|
172
|
-
|
|
173
|
-
|
|
197
|
+
const artifact = await this.materializer.materialize(mediaAsset, {
|
|
198
|
+
variant: 'original',
|
|
199
|
+
destination: imagesDir,
|
|
200
|
+
});
|
|
201
|
+
const key = `${mediaAsset.sourceRef?.pixivKind}:${mediaAsset.sourceRef?.sourceId}`;
|
|
202
|
+
const asset = assets.find((a) => `${a.kind}:${a.sourceId}` === key);
|
|
203
|
+
if (asset) {
|
|
204
|
+
asset.localPath = artifact.path;
|
|
205
|
+
asset.status = 'downloaded';
|
|
206
|
+
}
|
|
174
207
|
}
|
|
175
208
|
catch (error) {
|
|
176
|
-
|
|
177
|
-
asset
|
|
178
|
-
|
|
209
|
+
const key = `${mediaAsset.sourceRef?.pixivKind}:${mediaAsset.sourceRef?.sourceId}`;
|
|
210
|
+
const asset = assets.find((a) => `${a.kind}:${a.sourceId}` === key);
|
|
211
|
+
if (asset) {
|
|
212
|
+
asset.status = 'failed';
|
|
213
|
+
asset.failureReason = error instanceof Error ? error.message : String(error);
|
|
214
|
+
}
|
|
215
|
+
const assetId = mediaAsset.sourceRef?.sourceId ?? 'unknown';
|
|
216
|
+
logger_1.logger.warn(`Failed to download novel inline image ${assetId} for novel ${detail.id}`, {
|
|
179
217
|
novelId: detail.id,
|
|
180
|
-
sourceId:
|
|
181
|
-
reason:
|
|
218
|
+
sourceId: assetId,
|
|
219
|
+
reason: error instanceof Error ? error.message : String(error),
|
|
182
220
|
});
|
|
183
221
|
}
|
|
184
222
|
}
|
|
@@ -186,13 +224,22 @@ class NovelDownloader {
|
|
|
186
224
|
logger_1.logger.info(`Novel ${detail.id} inline images: ${assets.filter((a) => a.status === 'downloaded').length}/${assets.length} downloaded`, { novelId: detail.id });
|
|
187
225
|
}
|
|
188
226
|
}
|
|
227
|
+
const downloadByAssetKey = new Map(assets.filter((a) => a.status === 'downloaded' && a.localPath)
|
|
228
|
+
.map((a) => [`${a.kind}:${a.sourceId}`, a]));
|
|
189
229
|
// Rich-media markdown sidecar (RFC 1 Phase 2): same path as the .txt
|
|
190
230
|
// (compat format stays), inline images become relative 
|
|
191
231
|
// refs so a later TelePress/TelePost phase can render the Telegraph page.
|
|
192
232
|
let richMediaPath;
|
|
193
|
-
|
|
233
|
+
const hasDownloadedImages = assets.some((a) => a.status === 'downloaded');
|
|
234
|
+
// On-demand mode (no local images) still emits a resolvable md sidecar from
|
|
235
|
+
// media references so TelePress can render via proxy without a download.
|
|
236
|
+
const hasResolvablePending = assets.some((a) => a.status === 'pending' && Boolean(a.url));
|
|
237
|
+
if (hasDownloadedImages || hasResolvablePending) {
|
|
194
238
|
try {
|
|
195
|
-
const
|
|
239
|
+
const mdContent = hasDownloadedImages
|
|
240
|
+
? (0, novelMarkers_1.renderNovelMarkdown)(text, assets)
|
|
241
|
+
: (0, novelMarkers_1.renderNovelMarkdownReference)(text, assets);
|
|
242
|
+
const mdPath = await this.fileService.saveText(`${header}\n${mdContent}`, fileName.replace(/\.txt$/, '.md'), metadata);
|
|
196
243
|
richMediaPath = mdPath;
|
|
197
244
|
logger_1.logger.info(`Saved novel ${detail.id} rich-media markdown sidecar`, { filePath: mdPath });
|
|
198
245
|
}
|
|
@@ -289,12 +336,45 @@ class NovelDownloader {
|
|
|
289
336
|
filePath,
|
|
290
337
|
...(detectedLang ? { language: detectedLang.name, isChinese: detectedLang.isChinese } : {}),
|
|
291
338
|
});
|
|
339
|
+
const workId = String(detail.id);
|
|
340
|
+
const mediaAssets = [];
|
|
341
|
+
const artifacts = [];
|
|
342
|
+
if (filePath) {
|
|
343
|
+
artifacts.push({ id: (0, Artifact_1.artifactId)(workId, 'text', (0, node_path_1.basename)(filePath)), workId, variant: 'text', path: filePath });
|
|
344
|
+
}
|
|
345
|
+
if (richMediaPath) {
|
|
346
|
+
artifacts.push({ id: (0, Artifact_1.artifactId)(workId, 'markdown', (0, node_path_1.basename)(richMediaPath)), workId, variant: 'markdown', path: richMediaPath });
|
|
347
|
+
}
|
|
348
|
+
if (metadataPath) {
|
|
349
|
+
artifacts.push({ id: (0, Artifact_1.artifactId)(workId, 'metadata', (0, node_path_1.basename)(metadataPath)), workId, variant: 'metadata', path: metadataPath });
|
|
350
|
+
}
|
|
351
|
+
if (archivePath) {
|
|
352
|
+
artifacts.push({ id: (0, Artifact_1.artifactId)(workId, 'zip', (0, node_path_1.basename)(archivePath)), workId, variant: 'zip', path: archivePath });
|
|
353
|
+
}
|
|
354
|
+
for (const mediaAsset of resolved.mediaAssets) {
|
|
355
|
+
const a = downloadByAssetKey.get(`${mediaAsset.sourceRef?.pixivKind}:${mediaAsset.sourceRef?.sourceId}`);
|
|
356
|
+
if (!a?.localPath)
|
|
357
|
+
continue;
|
|
358
|
+
const imageArtifactId = (0, Artifact_1.artifactId)(workId, 'original', (0, node_path_1.basename)(a.localPath));
|
|
359
|
+
mediaAssets.push(buildMediaAssetById(mediaAsset, imageArtifactId));
|
|
360
|
+
artifacts.push({
|
|
361
|
+
id: imageArtifactId,
|
|
362
|
+
sourceAssetId: mediaAsset.id,
|
|
363
|
+
workId,
|
|
364
|
+
variant: 'original',
|
|
365
|
+
path: a.localPath,
|
|
366
|
+
});
|
|
367
|
+
}
|
|
368
|
+
// On-demand mode carries resolved references without local originals.
|
|
369
|
+
const returnedMediaAssets = mediaAssets.length ? mediaAssets : resolved.mediaAssets;
|
|
292
370
|
return {
|
|
293
|
-
pixivId:
|
|
371
|
+
pixivId: workId,
|
|
294
372
|
type: 'novel',
|
|
295
373
|
title: detail.title,
|
|
296
374
|
tags: tags.map((item) => item.name).filter(Boolean),
|
|
297
375
|
files: archivePath ? [filePath, archivePath] : [filePath],
|
|
376
|
+
mediaAssets: returnedMediaAssets,
|
|
377
|
+
artifacts,
|
|
298
378
|
cleanupFiles: metadataPath ? [metadataPath] : [],
|
|
299
379
|
spoiler: (detail.x_restrict ?? 0) > 0,
|
|
300
380
|
xRestrict: detail.x_restrict,
|
|
@@ -306,18 +386,7 @@ class NovelDownloader {
|
|
|
306
386
|
}
|
|
307
387
|
}
|
|
308
388
|
exports.NovelDownloader = NovelDownloader;
|
|
309
|
-
function
|
|
310
|
-
|
|
311
|
-
try {
|
|
312
|
-
const pathname = new URL(asset.url).pathname;
|
|
313
|
-
const last = pathname.split('/').pop() || '';
|
|
314
|
-
const dot = last.lastIndexOf('.');
|
|
315
|
-
if (dot >= 0)
|
|
316
|
-
ext = last.slice(dot);
|
|
317
|
-
}
|
|
318
|
-
catch {
|
|
319
|
-
// fall through: no extension
|
|
320
|
-
}
|
|
321
|
-
return `${asset.sourceId}${ext}` || asset.sourceId;
|
|
389
|
+
function buildMediaAssetById(mediaAsset, artifactIdValue) {
|
|
390
|
+
return { ...mediaAsset, artifactId: artifactIdValue };
|
|
322
391
|
}
|
|
323
392
|
//# sourceMappingURL=NovelDownloader.js.map
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { IPixivClient } from '../../interfaces/IPixivClient';
|
|
2
|
+
import { IFileService } from '../../interfaces/IFileService';
|
|
3
|
+
import { type MediaAsset } from '../../domain/media/MediaAsset';
|
|
4
|
+
import { type Artifact } from '../../domain/media/Artifact';
|
|
5
|
+
export interface MaterializationOptions {
|
|
6
|
+
variant?: 'original' | 'delivery';
|
|
7
|
+
destination?: string;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* Turns a canonical MediaAsset into a local Artifact on demand. This is the
|
|
11
|
+
* future lazy-materialization boundary; today it wraps the existing
|
|
12
|
+
* IPixivClient.downloadImage + IFileService.saveBinary path so no new HTTP
|
|
13
|
+
* download implementation exists.
|
|
14
|
+
*/
|
|
15
|
+
export interface MediaMaterializer {
|
|
16
|
+
materialize(asset: MediaAsset, options?: MaterializationOptions): Promise<Artifact>;
|
|
17
|
+
}
|
|
18
|
+
export declare class PixivMediaMaterializer implements MediaMaterializer {
|
|
19
|
+
private readonly client;
|
|
20
|
+
private readonly fileService;
|
|
21
|
+
constructor(client: IPixivClient, fileService: IFileService);
|
|
22
|
+
materialize(asset: MediaAsset, options?: MaterializationOptions): Promise<Artifact>;
|
|
23
|
+
}
|
|
24
|
+
//# sourceMappingURL=MediaMaterializer.d.ts.map
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.PixivMediaMaterializer = void 0;
|
|
4
|
+
const Artifact_1 = require("../../domain/media/Artifact");
|
|
5
|
+
function fileNameFor(asset) {
|
|
6
|
+
const sourceId = asset.sourceRef?.sourceId ?? '';
|
|
7
|
+
let ext = '';
|
|
8
|
+
try {
|
|
9
|
+
const last = new URL(asset.sourceUrl).pathname.split('/').pop() || '';
|
|
10
|
+
const dot = last.lastIndexOf('.');
|
|
11
|
+
if (dot >= 0)
|
|
12
|
+
ext = last.slice(dot);
|
|
13
|
+
}
|
|
14
|
+
catch {
|
|
15
|
+
// keep no extension
|
|
16
|
+
}
|
|
17
|
+
return `${sourceId}${ext}` || 'image';
|
|
18
|
+
}
|
|
19
|
+
class PixivMediaMaterializer {
|
|
20
|
+
client;
|
|
21
|
+
fileService;
|
|
22
|
+
constructor(client, fileService) {
|
|
23
|
+
this.client = client;
|
|
24
|
+
this.fileService = fileService;
|
|
25
|
+
}
|
|
26
|
+
async materialize(asset, options = {}) {
|
|
27
|
+
if (!options.destination) {
|
|
28
|
+
throw new Error('PixivMediaMaterializer requires options.destination');
|
|
29
|
+
}
|
|
30
|
+
const buffer = await this.client.downloadImage(asset.sourceUrl);
|
|
31
|
+
const fileName = fileNameFor(asset);
|
|
32
|
+
const path = await this.fileService.saveBinary(buffer, fileName, options.destination);
|
|
33
|
+
const variant = options.variant ?? 'original';
|
|
34
|
+
return {
|
|
35
|
+
id: (0, Artifact_1.artifactId)(asset.sourceRef?.workId ?? '', variant, fileName),
|
|
36
|
+
sourceAssetId: asset.id,
|
|
37
|
+
workId: asset.sourceRef?.workId ?? '',
|
|
38
|
+
variant,
|
|
39
|
+
path,
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
exports.PixivMediaMaterializer = PixivMediaMaterializer;
|
|
44
|
+
//# sourceMappingURL=MediaMaterializer.js.map
|
|
@@ -75,4 +75,18 @@ export declare function demo(): void;
|
|
|
75
75
|
* original text so nothing is silently dropped (partial success).
|
|
76
76
|
*/
|
|
77
77
|
export declare function renderNovelMarkdown(source: string, assets: NovelAsset[]): string;
|
|
78
|
+
/**
|
|
79
|
+
* Stable relative filename for an inline novel media reference without a local
|
|
80
|
+
* download. Uses the sourceId plus the URL extension so the manifest can point
|
|
81
|
+
* at a proxy URL while the rendered Markdown still resolves to
|
|
82
|
+
* `images/<sourceId>.jpg`.
|
|
83
|
+
*/
|
|
84
|
+
export declare function novelReferenceFileName(url: string | undefined, sourceId: string): string;
|
|
85
|
+
/**
|
|
86
|
+
* Markdown sidecar for on-demand (non-downloading) previews. Renders
|
|
87
|
+
* `` for every resolvable asset so the manifest can be
|
|
88
|
+
* rehydrated by TelePress via proxy; markers without a resolvable URL stay as
|
|
89
|
+
* their original text.
|
|
90
|
+
*/
|
|
91
|
+
export declare function renderNovelMarkdownReference(source: string, assets: NovelAsset[]): string;
|
|
78
92
|
//# sourceMappingURL=novelMarkers.d.ts.map
|
|
@@ -4,6 +4,8 @@ exports.scanNovelMarkers = scanNovelMarkers;
|
|
|
4
4
|
exports.extractNovelAssets = extractNovelAssets;
|
|
5
5
|
exports.demo = demo;
|
|
6
6
|
exports.renderNovelMarkdown = renderNovelMarkdown;
|
|
7
|
+
exports.novelReferenceFileName = novelReferenceFileName;
|
|
8
|
+
exports.renderNovelMarkdownReference = renderNovelMarkdownReference;
|
|
7
9
|
const node_path_1 = require("node:path");
|
|
8
10
|
const linkRegex = /https?:\/\/\S+/;
|
|
9
11
|
function parseMarker(span) {
|
|
@@ -141,4 +143,50 @@ function renderNovelMarkdown(source, assets) {
|
|
|
141
143
|
})
|
|
142
144
|
.join('');
|
|
143
145
|
}
|
|
146
|
+
/**
|
|
147
|
+
* Stable relative filename for an inline novel media reference without a local
|
|
148
|
+
* download. Uses the sourceId plus the URL extension so the manifest can point
|
|
149
|
+
* at a proxy URL while the rendered Markdown still resolves to
|
|
150
|
+
* `images/<sourceId>.jpg`.
|
|
151
|
+
*/
|
|
152
|
+
function novelReferenceFileName(url, sourceId) {
|
|
153
|
+
let ext = '';
|
|
154
|
+
if (url) {
|
|
155
|
+
try {
|
|
156
|
+
const path = new URL(url).pathname;
|
|
157
|
+
const match = /\.(jpg|jpeg|png|gif|webp)$/i.exec(path);
|
|
158
|
+
if (match)
|
|
159
|
+
ext = match[1].toLowerCase();
|
|
160
|
+
}
|
|
161
|
+
catch {
|
|
162
|
+
// non-URL fall through to default ext
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
const safeId = sourceId.replace(/[^\w.-]+/g, '_');
|
|
166
|
+
return `images/${safeId}${ext ? `.${ext}` : '.jpg'}`;
|
|
167
|
+
}
|
|
168
|
+
/**
|
|
169
|
+
* Markdown sidecar for on-demand (non-downloading) previews. Renders
|
|
170
|
+
* `` for every resolvable asset so the manifest can be
|
|
171
|
+
* rehydrated by TelePress via proxy; markers without a resolvable URL stay as
|
|
172
|
+
* their original text.
|
|
173
|
+
*/
|
|
174
|
+
function renderNovelMarkdownReference(source, assets) {
|
|
175
|
+
return scanNovelMarkers(source)
|
|
176
|
+
.map((marker) => {
|
|
177
|
+
if (marker.type === 'text')
|
|
178
|
+
return marker.value;
|
|
179
|
+
const asset = assets.find((a) => a.marker === marker.raw);
|
|
180
|
+
if (!asset)
|
|
181
|
+
return marker.raw;
|
|
182
|
+
if (asset.status === 'downloaded' && asset.localPath) {
|
|
183
|
+
return `(asset.localPath)})`;
|
|
184
|
+
}
|
|
185
|
+
if (asset.status === 'pending' && asset.url) {
|
|
186
|
+
return `})`;
|
|
187
|
+
}
|
|
188
|
+
return marker.raw;
|
|
189
|
+
})
|
|
190
|
+
.join('');
|
|
191
|
+
}
|
|
144
192
|
//# sourceMappingURL=novelMarkers.js.map
|
package/dist/package.json
CHANGED
package/dist/version.js
CHANGED
|
@@ -2,5 +2,5 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.BUILD = void 0;
|
|
4
4
|
// GENERATED by scripts/write-version.js — do not edit manually.
|
|
5
|
-
exports.BUILD = { version: '2.
|
|
5
|
+
exports.BUILD = { version: '2.40.0', commit: '8d72f5f5761a' };
|
|
6
6
|
//# sourceMappingURL=version.js.map
|
package/dist/webui/package.json
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pixivflow",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.40.0",
|
|
4
4
|
"description": "🎨 Pixiv 下载、筛选与自动收集工具 - 批量下载插画和小说、按标签/热度/日期筛选、定时任务与可靠 HTTP 交付 | Pixiv downloader and automation toolkit with filtering, scheduling and reliable HTTP delivery",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|