socialdatax-skills 0.2.25 → 0.2.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -3
- package/cli.mjs +59 -13
- package/lib/media/bilibili-download.mjs +73 -8
- package/lib/media/xhs-download.mjs +723 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
This public package provides the unified skill installer and direct CLI helper for SocialDataX services.
|
|
4
4
|
|
|
5
|
-
The current public tools support 小红书 / Xiaohongshu / XHS / RedNote, 抖音 / Douyin, 快手 / Kuaishou / Kwai, 微博 / Weibo, 视频号 / WeChat Channels, and 微信公众号 / WeChat Official Account article content research and analysis workflows, plus Bilibili / 哔哩哔哩 / B站 local video download and 敏感词检测 / 违禁词检查 text checks. The public skill layer is intentionally named by capability so supported services can evolve without changing the installation model.
|
|
5
|
+
The current public tools support 小红书 / Xiaohongshu / XHS / RedNote, 抖音 / Douyin, 快手 / Kuaishou / Kwai, 微博 / Weibo, 视频号 / WeChat Channels, and 微信公众号 / WeChat Official Account article content research and analysis workflows, plus XHS local media download, Bilibili / 哔哩哔哩 / B站 local video download, and 敏感词检测 / 违禁词检查 text checks. The public skill layer is intentionally named by capability so supported services can evolve without changing the installation model.
|
|
6
6
|
|
|
7
7
|
- direct `npx` JSON commands for agents that can run shell commands
|
|
8
8
|
- AgentSkills-compatible installers split by capability for OpenClaw, Hermes Agent, Codex, Claude Code, and general agent skill directories
|
|
@@ -28,6 +28,9 @@ Common search phrases for this skill package:
|
|
|
28
28
|
- `Xiaohongshu skills`
|
|
29
29
|
- `XHS skills`
|
|
30
30
|
- `RedNote skills`
|
|
31
|
+
- `XHS media download`
|
|
32
|
+
- `小红书图片下载`
|
|
33
|
+
- `小红书视频下载`
|
|
31
34
|
- `Douyin skills`
|
|
32
35
|
- `抖音 skills`
|
|
33
36
|
- `Kuaishou skills`
|
|
@@ -79,7 +82,7 @@ Common search phrases for this skill package:
|
|
|
79
82
|
- Sensitive Words Check MCP server name: `sensitive-check`
|
|
80
83
|
- Sensitive Words Check future MCP registry name: `com.socialdatax/sensitive-check`
|
|
81
84
|
- Unified MCP registry name: none; this package installs skills and calls explicit platform services.
|
|
82
|
-
- Current public capability version: `0.2.
|
|
85
|
+
- Current public capability version: `0.2.27`
|
|
83
86
|
|
|
84
87
|
## Direct CLI
|
|
85
88
|
|
|
@@ -102,6 +105,7 @@ npx -y socialdatax-skills@latest xhs user-posts --user-id "<user_id>" --since-da
|
|
|
102
105
|
npx -y socialdatax-skills@latest xhs transcript --url "<note_url_or_share_text>" --pretty
|
|
103
106
|
npx -y socialdatax-skills@latest xhs transcript --note-id "<note_id>" --pretty
|
|
104
107
|
npx -y socialdatax-skills@latest xhs transcript --job-id "<job_id>" --pretty
|
|
108
|
+
npx -y socialdatax-skills@latest xhs download-media --url "<xhs_media_url>" --output-dir ./downloads --pretty
|
|
105
109
|
npx -y socialdatax-skills@latest douyin hot-search --pretty
|
|
106
110
|
npx -y socialdatax-skills@latest douyin search --keyword "露营" --pretty
|
|
107
111
|
npx -y socialdatax-skills@latest douyin detail --aweme-id "<aweme_id>" --pretty
|
|
@@ -170,7 +174,7 @@ npx -y socialdatax-skills@latest wechat transcript --job-id "<job_id>" --pretty
|
|
|
170
174
|
npx -y socialdatax-skills@latest sensitive-check text --text "<content>" --platform xhs --pretty
|
|
171
175
|
```
|
|
172
176
|
|
|
173
|
-
Most direct CLI commands print a JSON envelope with `platform`, `tool`, `arguments`, and `data`. `sensitive-check` prints `platform`, `tool`, and `data` only; it does not echo the original text in CLI output. `wechat decrypt-media` is a local save command: pass the `video.video_url` returned by WeChat detail and an `--output` file path. It saves the media locally, decrypts when needed, and does not require `SOCIALDATAX_API_KEY`.
|
|
177
|
+
Most direct CLI commands print a JSON envelope with `platform`, `tool`, `arguments`, and `data`. `sensitive-check` prints `platform`, `tool`, and `data` only; it does not echo the original text in CLI output. `wechat decrypt-media` is a local save command: pass the `video.video_url` returned by WeChat detail and an `--output` file path. It saves the media locally, decrypts when needed, and does not require `SOCIALDATAX_API_KEY`. `xhs download-media` is also local: pass one image or video media URL returned by XHS detail and either `--output <file>` or `--output-dir <directory>`. It writes through a `.part` file, resumes partial downloads when the server supports range requests, skips an already existing output file, and infers common image/video extensions in `--output-dir` mode.
|
|
174
178
|
|
|
175
179
|
`bilibili download` calls SocialDataX once to fetch short-lived download links, then downloads video and audio tracks on the local machine and merges them with local `ffmpeg`. Set `SOCIALDATAX_API_KEY`, install `ffmpeg`, and pass either `--output <file>` or `--output-dir <directory>`. The download-links request consumes 10 credits; the local track download and merge do not consume additional SocialDataX credits. Use `--ffmpeg-path <path>` when `ffmpeg` is not on `PATH`, and `--keep-tracks` to retain the separate track files after merge.
|
|
176
180
|
|
package/cli.mjs
CHANGED
|
@@ -12,13 +12,17 @@ import { homedir } from "node:os";
|
|
|
12
12
|
import { dirname, join, resolve } from "node:path";
|
|
13
13
|
import { fileURLToPath } from "node:url";
|
|
14
14
|
|
|
15
|
-
import {
|
|
15
|
+
import {
|
|
16
|
+
assertBilibiliFfmpegAvailable,
|
|
17
|
+
downloadBilibiliVideoFromManifest,
|
|
18
|
+
} from "./lib/media/bilibili-download.mjs";
|
|
16
19
|
import { decryptWechatMediaCommand } from "./lib/media/wechat-decrypt.mjs";
|
|
20
|
+
import { downloadXhsMediaFromUrl } from "./lib/media/xhs-download.mjs";
|
|
17
21
|
|
|
18
22
|
export { decryptWechatMediaCommand };
|
|
19
23
|
|
|
20
24
|
const PACKAGE_NAME = "socialdatax-skills";
|
|
21
|
-
const PACKAGE_VERSION = "0.2.
|
|
25
|
+
const PACKAGE_VERSION = "0.2.27";
|
|
22
26
|
const PACKAGE_SPEC = `${PACKAGE_NAME}@latest`;
|
|
23
27
|
const LOG_PREFIX = `[${PACKAGE_NAME}]`;
|
|
24
28
|
const MIN_NODE_VERSION = "20.18.1";
|
|
@@ -160,6 +164,7 @@ const XHS_DIRECT_ACTION_OPTIONS = {
|
|
|
160
164
|
"pretty",
|
|
161
165
|
],
|
|
162
166
|
transcript: ["url", "noteId", "jobId", "pretty"],
|
|
167
|
+
"download-media": ["url", "output", "outputDir", "pretty"],
|
|
163
168
|
};
|
|
164
169
|
const XHS_DIRECT_ACTION_NAMES = Object.keys(XHS_DIRECT_ACTION_OPTIONS).join(", ");
|
|
165
170
|
const XHS_OPTION_DISPLAY_NAMES = {
|
|
@@ -179,6 +184,8 @@ const XHS_OPTION_DISPLAY_NAMES = {
|
|
|
179
184
|
profileUrl: "--profile-url",
|
|
180
185
|
userId: "--user-id",
|
|
181
186
|
jobId: "--job-id",
|
|
187
|
+
output: "--output",
|
|
188
|
+
outputDir: "--output-dir",
|
|
182
189
|
};
|
|
183
190
|
const XHS_SEARCH_SORT_TYPES = [
|
|
184
191
|
"general",
|
|
@@ -527,12 +534,12 @@ const PLATFORMS = {
|
|
|
527
534
|
{
|
|
528
535
|
name: "xhs_get_note_comments_by_note_id",
|
|
529
536
|
description:
|
|
530
|
-
"Fetch paginated first-level comments by note ID
|
|
537
|
+
"Fetch paginated first-level comments by note ID.",
|
|
531
538
|
},
|
|
532
539
|
{
|
|
533
540
|
name: "xhs_get_note_comments_by_note_url",
|
|
534
541
|
description:
|
|
535
|
-
"Fetch paginated first-level comments from a note URL, short link, or share text
|
|
542
|
+
"Fetch paginated first-level comments from a note URL, short link, or share text.",
|
|
536
543
|
},
|
|
537
544
|
{
|
|
538
545
|
name: "xhs_get_note_sub_comments_by_comment_id",
|
|
@@ -1094,6 +1101,26 @@ function validateXhsDirectActionOptions(action, options) {
|
|
|
1094
1101
|
return;
|
|
1095
1102
|
}
|
|
1096
1103
|
|
|
1104
|
+
if (action === "download-media") {
|
|
1105
|
+
validateKnownOptions(options, allowedOptions);
|
|
1106
|
+
validateFlagOption(options, "pretty", "--pretty");
|
|
1107
|
+
for (const key of allowedOptions) {
|
|
1108
|
+
if (!DIRECT_BOOLEAN_OPTIONS.has(key)) {
|
|
1109
|
+
requireOptionValue(options, key, XHS_OPTION_DISPLAY_NAMES[key]);
|
|
1110
|
+
}
|
|
1111
|
+
}
|
|
1112
|
+
if (!options.url) {
|
|
1113
|
+
throw new Error("Missing --url for xhs download-media.");
|
|
1114
|
+
}
|
|
1115
|
+
if (!options.output && !options.outputDir) {
|
|
1116
|
+
throw new Error("Missing --output or --output-dir for xhs download-media.");
|
|
1117
|
+
}
|
|
1118
|
+
if (options.output && options.outputDir) {
|
|
1119
|
+
throw new Error("Use only one of --output or --output-dir for xhs download-media.");
|
|
1120
|
+
}
|
|
1121
|
+
return;
|
|
1122
|
+
}
|
|
1123
|
+
|
|
1097
1124
|
validateKnownOptions(options, allowedDirectOptions(allowedOptions));
|
|
1098
1125
|
validateDirectMetaOptions(options);
|
|
1099
1126
|
validateDirectPaginationOptions("xhs", action, options);
|
|
@@ -1844,6 +1871,9 @@ function printHelp() {
|
|
|
1844
1871
|
console.log(` npx -y ${PACKAGE_SPEC} xhs transcript --url "<xhs_note_url_or_share_text>" --pretty`);
|
|
1845
1872
|
console.log(" Submit or check an XHS video note speech-to-text transcript job.");
|
|
1846
1873
|
console.log("");
|
|
1874
|
+
console.log(` npx -y ${PACKAGE_SPEC} xhs download-media --url "<xhs_media_url>" --output-dir ./downloads --pretty`);
|
|
1875
|
+
console.log(" Save one XHS image or video media URL returned by detail to a local file.");
|
|
1876
|
+
console.log("");
|
|
1847
1877
|
console.log(` npx -y ${PACKAGE_SPEC} douyin hot-search --pretty`);
|
|
1848
1878
|
console.log(" Call the Douyin main hot search list tool directly and print JSON.");
|
|
1849
1879
|
console.log("");
|
|
@@ -2087,13 +2117,14 @@ function printHelp() {
|
|
|
2087
2117
|
console.log(" --keyword <text>");
|
|
2088
2118
|
console.log(" --url <url-or-share-text>");
|
|
2089
2119
|
console.log(" Content link, short link, or share text for URL-based detail/comment/article commands.");
|
|
2120
|
+
console.log(" For xhs download-media, pass one image or video media URL returned by XHS detail.");
|
|
2090
2121
|
console.log(" For Douyin detail/comments, pass a content page link, not video.play_url.");
|
|
2091
2122
|
console.log(" --media-url <video.video_url>");
|
|
2092
2123
|
console.log(" WeChat detail result media URL for local decrypt-media download.");
|
|
2093
2124
|
console.log(" --output <file>");
|
|
2094
|
-
console.log(" Output file path for local decrypt-media or Bilibili download.");
|
|
2125
|
+
console.log(" Output file path for local decrypt-media, XHS download-media, or Bilibili download.");
|
|
2095
2126
|
console.log(" --output-dir <directory>");
|
|
2096
|
-
console.log(" Directory for Bilibili download output when --output is omitted.");
|
|
2127
|
+
console.log(" Directory for XHS download-media or Bilibili download output when --output is omitted.");
|
|
2097
2128
|
console.log(" --ffmpeg-path <path>");
|
|
2098
2129
|
console.log(" ffmpeg executable path for Bilibili download; defaults to ffmpeg.");
|
|
2099
2130
|
console.log(" --keep-tracks");
|
|
@@ -2215,6 +2246,13 @@ async function runXhsDirectCommand(args) {
|
|
|
2215
2246
|
}
|
|
2216
2247
|
validateXhsDirectActionOptions(action, options);
|
|
2217
2248
|
|
|
2249
|
+
if (action === "download-media") {
|
|
2250
|
+
const data = await downloadXhsMediaFromUrl(options.url, options);
|
|
2251
|
+
process.stdout.write(JSON.stringify(data, null, options.pretty ? 2 : 0));
|
|
2252
|
+
process.stdout.write("\n");
|
|
2253
|
+
return;
|
|
2254
|
+
}
|
|
2255
|
+
|
|
2218
2256
|
const operation = attachDirectMetadata(buildXhsOperation(action, options), options);
|
|
2219
2257
|
const data = shouldUsePaginatedDirectOutput(options)
|
|
2220
2258
|
? await callPaginatedDirectOperation(operation, options)
|
|
@@ -2359,6 +2397,10 @@ async function assertBilibiliDownloadLocalPreflight(options) {
|
|
|
2359
2397
|
await assertBilibiliParentDirectory(outputDir, "--output-dir parent path");
|
|
2360
2398
|
}
|
|
2361
2399
|
}
|
|
2400
|
+
|
|
2401
|
+
ensureSupportedNodeVersion();
|
|
2402
|
+
readDirectApiKey(PLATFORMS.bilibili);
|
|
2403
|
+
await assertBilibiliFfmpegAvailable(options.ffmpegPath || "ffmpeg");
|
|
2362
2404
|
}
|
|
2363
2405
|
|
|
2364
2406
|
async function assertBilibiliParentDirectory(targetPath, displayName) {
|
|
@@ -4203,13 +4245,7 @@ function buildRepliesOperationForComment(platform, comment, parentArguments, par
|
|
|
4203
4245
|
async function callMcpBackend(operation) {
|
|
4204
4246
|
ensureSupportedNodeVersion();
|
|
4205
4247
|
const { platform, tool } = operation;
|
|
4206
|
-
const apiKey =
|
|
4207
|
-
if (!apiKey) {
|
|
4208
|
-
throw new Error(
|
|
4209
|
-
`Missing API Key. Set ${PRIMARY_API_KEY_ENV} before running direct CLI calls.`
|
|
4210
|
-
);
|
|
4211
|
-
}
|
|
4212
|
-
|
|
4248
|
+
const apiKey = readDirectApiKey(platform);
|
|
4213
4249
|
const { Client, StreamableHTTPClientTransport } = await loadMcpSdkModules();
|
|
4214
4250
|
const upstreamUrl = resolveUpstreamUrl(platform);
|
|
4215
4251
|
const client = new Client(
|
|
@@ -4258,6 +4294,16 @@ async function callMcpBackend(operation) {
|
|
|
4258
4294
|
}
|
|
4259
4295
|
}
|
|
4260
4296
|
|
|
4297
|
+
function readDirectApiKey(platform) {
|
|
4298
|
+
const apiKey = readFirstEnv(platform.apiKeyEnv);
|
|
4299
|
+
if (!apiKey) {
|
|
4300
|
+
throw new Error(
|
|
4301
|
+
`Missing API Key. Set ${PRIMARY_API_KEY_ENV} before running direct CLI calls.`
|
|
4302
|
+
);
|
|
4303
|
+
}
|
|
4304
|
+
return apiKey;
|
|
4305
|
+
}
|
|
4306
|
+
|
|
4261
4307
|
async function loadMcpSdkModules() {
|
|
4262
4308
|
if (!mcpSdkModules) {
|
|
4263
4309
|
const [{ Client }, { StreamableHTTPClientTransport }] = await Promise.all([
|
|
@@ -8,7 +8,12 @@ import {
|
|
|
8
8
|
stat,
|
|
9
9
|
unlink,
|
|
10
10
|
} from "node:fs/promises";
|
|
11
|
-
import {
|
|
11
|
+
import {
|
|
12
|
+
arch as osArch,
|
|
13
|
+
homedir,
|
|
14
|
+
platform as osPlatform,
|
|
15
|
+
release as osRelease,
|
|
16
|
+
} from "node:os";
|
|
12
17
|
import { dirname, join, resolve } from "node:path";
|
|
13
18
|
|
|
14
19
|
import {
|
|
@@ -39,11 +44,18 @@ export async function downloadBilibiliVideoFromManifest(
|
|
|
39
44
|
validateBilibiliDownloadManifest(manifest);
|
|
40
45
|
|
|
41
46
|
const outputPath = resolveBilibiliOutputPath(manifest, options);
|
|
47
|
+
const ffmpegPath = options.ffmpegPath || "ffmpeg";
|
|
42
48
|
await mkdir(dirname(outputPath), { recursive: true });
|
|
43
49
|
await reserveBilibiliOutputPath(outputPath);
|
|
44
50
|
|
|
45
51
|
const downloadedTracks = [];
|
|
46
52
|
try {
|
|
53
|
+
await assertBilibiliTrackPathsAvailable(
|
|
54
|
+
manifest.download_manifest.tracks,
|
|
55
|
+
outputPath
|
|
56
|
+
);
|
|
57
|
+
await assertBilibiliFfmpegAvailable(ffmpegPath, spawnProcess);
|
|
58
|
+
|
|
47
59
|
for (const track of manifest.download_manifest.tracks) {
|
|
48
60
|
downloadedTracks.push(
|
|
49
61
|
await downloadBilibiliTrack(track, {
|
|
@@ -63,7 +75,7 @@ export async function downloadBilibiliVideoFromManifest(
|
|
|
63
75
|
}
|
|
64
76
|
|
|
65
77
|
await mergeBilibiliTracksWithFfmpeg({
|
|
66
|
-
ffmpegPath
|
|
78
|
+
ffmpegPath,
|
|
67
79
|
videoPath: videoTrack.path,
|
|
68
80
|
audioPath: audioTrack.path,
|
|
69
81
|
outputPath,
|
|
@@ -222,9 +234,10 @@ async function downloadBilibiliTrack(
|
|
|
222
234
|
maxDownloadBytes,
|
|
223
235
|
}
|
|
224
236
|
) {
|
|
225
|
-
const trackType =
|
|
226
|
-
|
|
227
|
-
|
|
237
|
+
const { trackType, ext, trackPath } = resolveBilibiliTrackTarget(
|
|
238
|
+
track,
|
|
239
|
+
outputPath
|
|
240
|
+
);
|
|
228
241
|
const tempPath = `${trackPath}.tmp-${process.pid}-${Date.now()}-${randomUUID()}`;
|
|
229
242
|
await mkdir(dirname(trackPath), { recursive: true });
|
|
230
243
|
|
|
@@ -275,6 +288,31 @@ async function downloadBilibiliTrack(
|
|
|
275
288
|
};
|
|
276
289
|
}
|
|
277
290
|
|
|
291
|
+
async function assertBilibiliTrackPathsAvailable(tracks, outputPath) {
|
|
292
|
+
for (const track of tracks) {
|
|
293
|
+
const { trackType, trackPath } = resolveBilibiliTrackTarget(track, outputPath);
|
|
294
|
+
const existingTrack = await stat(trackPath).catch((error) => {
|
|
295
|
+
if (error?.code === "ENOENT") {
|
|
296
|
+
return undefined;
|
|
297
|
+
}
|
|
298
|
+
throw error;
|
|
299
|
+
});
|
|
300
|
+
if (existingTrack) {
|
|
301
|
+
throw new Error(`Bilibili ${trackType} track file already exists.`);
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
function resolveBilibiliTrackTarget(track, outputPath) {
|
|
307
|
+
const trackType = String(track.type);
|
|
308
|
+
const ext = sanitizeFileExtension(track.ext || "m4s");
|
|
309
|
+
return {
|
|
310
|
+
trackType,
|
|
311
|
+
ext,
|
|
312
|
+
trackPath: `${outputPath}.${trackType}.${ext}`,
|
|
313
|
+
};
|
|
314
|
+
}
|
|
315
|
+
|
|
278
316
|
async function reserveBilibiliTrackPath(trackPath, trackType) {
|
|
279
317
|
let file;
|
|
280
318
|
try {
|
|
@@ -341,7 +379,21 @@ async function mergeBilibiliTracksWithFfmpeg({
|
|
|
341
379
|
await runFfmpegProcess(ffmpegPath, args, spawnProcess);
|
|
342
380
|
}
|
|
343
381
|
|
|
344
|
-
async function
|
|
382
|
+
export async function assertBilibiliFfmpegAvailable(
|
|
383
|
+
ffmpegPath = "ffmpeg",
|
|
384
|
+
spawnProcess = spawn
|
|
385
|
+
) {
|
|
386
|
+
await runFfmpegProcess(ffmpegPath, ["-version"], spawnProcess, {
|
|
387
|
+
failurePrefix: "ffmpeg availability check failed",
|
|
388
|
+
});
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
async function runFfmpegProcess(
|
|
392
|
+
ffmpegPath,
|
|
393
|
+
args,
|
|
394
|
+
spawnProcess,
|
|
395
|
+
{ failurePrefix = "ffmpeg failed for bilibili download" } = {}
|
|
396
|
+
) {
|
|
345
397
|
return new Promise((resolvePromise, reject) => {
|
|
346
398
|
let stderr = "";
|
|
347
399
|
const child = spawnProcess(ffmpegPath, args, {
|
|
@@ -358,7 +410,7 @@ async function runFfmpegProcess(ffmpegPath, args, spawnProcess) {
|
|
|
358
410
|
if (error?.code === "ENOENT") {
|
|
359
411
|
reject(
|
|
360
412
|
new Error(
|
|
361
|
-
|
|
413
|
+
buildFfmpegMissingMessage(ffmpegPath)
|
|
362
414
|
)
|
|
363
415
|
);
|
|
364
416
|
return;
|
|
@@ -372,13 +424,26 @@ async function runFfmpegProcess(ffmpegPath, args, spawnProcess) {
|
|
|
372
424
|
}
|
|
373
425
|
reject(
|
|
374
426
|
new Error(
|
|
375
|
-
|
|
427
|
+
`${failurePrefix} with exit code ${status}. ${stderr.trim()}`.trim()
|
|
376
428
|
)
|
|
377
429
|
);
|
|
378
430
|
});
|
|
379
431
|
});
|
|
380
432
|
}
|
|
381
433
|
|
|
434
|
+
function buildFfmpegMissingMessage(ffmpegPath) {
|
|
435
|
+
const detectedSystem = `${osPlatform()} ${osRelease()} ${osArch()}`;
|
|
436
|
+
return [
|
|
437
|
+
`ffmpeg not found at "${ffmpegPath}".`,
|
|
438
|
+
`Detected system: ${detectedSystem}.`,
|
|
439
|
+
"Install ffmpeg, then retry:",
|
|
440
|
+
"macOS (Homebrew): brew install ffmpeg",
|
|
441
|
+
"Ubuntu/Debian: sudo apt update && sudo apt install ffmpeg",
|
|
442
|
+
"Windows (winget): winget install Gyan.FFmpeg",
|
|
443
|
+
"If ffmpeg is already installed, pass --ffmpeg-path /path/to/ffmpeg for bilibili download.",
|
|
444
|
+
].join("\n");
|
|
445
|
+
}
|
|
446
|
+
|
|
382
447
|
function stringValue(value) {
|
|
383
448
|
return typeof value === "string" && value.trim() ? value : undefined;
|
|
384
449
|
}
|
|
@@ -0,0 +1,723 @@
|
|
|
1
|
+
import { constants as fsConstants } from "node:fs";
|
|
2
|
+
import {
|
|
3
|
+
copyFile as copyFilePath,
|
|
4
|
+
link as linkPath,
|
|
5
|
+
open as openFile,
|
|
6
|
+
mkdir,
|
|
7
|
+
stat,
|
|
8
|
+
unlink,
|
|
9
|
+
} from "node:fs/promises";
|
|
10
|
+
import { homedir } from "node:os";
|
|
11
|
+
import { basename, dirname, extname, join, resolve } from "node:path";
|
|
12
|
+
|
|
13
|
+
import {
|
|
14
|
+
MEDIA_DOWNLOAD_MAX_BYTES,
|
|
15
|
+
MEDIA_DOWNLOAD_TIMEOUT_MS,
|
|
16
|
+
assertMediaDownloadWithinLimit,
|
|
17
|
+
createMediaDownloadAbortSignal,
|
|
18
|
+
isAbortLikeError,
|
|
19
|
+
parseOptionalIntegerHeader,
|
|
20
|
+
writeAllToFile,
|
|
21
|
+
} from "./common.mjs";
|
|
22
|
+
|
|
23
|
+
const XHS_DOWNLOAD_USER_AGENT =
|
|
24
|
+
"Mozilla/5.0 (compatible; SocialDataX/1.0; +https://socialdatax.com)";
|
|
25
|
+
const XHS_DOWNLOAD_REFERER = "https://www.xiaohongshu.com/";
|
|
26
|
+
const XHS_DOWNLOAD_MAX_ATTEMPTS = 10;
|
|
27
|
+
const XHS_DOWNLOAD_RETRY_DELAY_MS = 250;
|
|
28
|
+
const XHS_RETRYABLE_HTTP_STATUSES = new Set([
|
|
29
|
+
408,
|
|
30
|
+
409,
|
|
31
|
+
425,
|
|
32
|
+
429,
|
|
33
|
+
500,
|
|
34
|
+
502,
|
|
35
|
+
503,
|
|
36
|
+
504,
|
|
37
|
+
]);
|
|
38
|
+
const XHS_EXPIRED_HTTP_STATUSES = new Set([403, 404, 410]);
|
|
39
|
+
const XHS_CONTENT_TYPE_EXTENSIONS = new Map([
|
|
40
|
+
["image/jpeg", "jpg"],
|
|
41
|
+
["image/jpg", "jpg"],
|
|
42
|
+
["image/png", "png"],
|
|
43
|
+
["image/webp", "webp"],
|
|
44
|
+
["image/heif", "heif"],
|
|
45
|
+
["image/heic", "heic"],
|
|
46
|
+
["video/mp4", "mp4"],
|
|
47
|
+
]);
|
|
48
|
+
const XHS_URL_FORMAT_EXTENSIONS = new Map([
|
|
49
|
+
["jpg", "jpg"],
|
|
50
|
+
["jpeg", "jpg"],
|
|
51
|
+
["png", "png"],
|
|
52
|
+
["webp", "webp"],
|
|
53
|
+
["heif", "heif"],
|
|
54
|
+
["heic", "heic"],
|
|
55
|
+
["mp4", "mp4"],
|
|
56
|
+
]);
|
|
57
|
+
export async function downloadXhsMediaFromUrl(
|
|
58
|
+
url,
|
|
59
|
+
options,
|
|
60
|
+
{
|
|
61
|
+
fetchMedia = fetch,
|
|
62
|
+
downloadTimeoutMs = MEDIA_DOWNLOAD_TIMEOUT_MS,
|
|
63
|
+
maxDownloadBytes = MEDIA_DOWNLOAD_MAX_BYTES,
|
|
64
|
+
retryDelayMs = XHS_DOWNLOAD_RETRY_DELAY_MS,
|
|
65
|
+
now = () => new Date(),
|
|
66
|
+
linkFile = linkPath,
|
|
67
|
+
copyFile = copyFilePath,
|
|
68
|
+
} = {}
|
|
69
|
+
) {
|
|
70
|
+
const parsedUrl = parseHttpUrl(url);
|
|
71
|
+
const target = await resolveXhsMediaTarget(parsedUrl, options, now);
|
|
72
|
+
const existingOutput = await stat(target.outputPath).catch(ignoreMissing);
|
|
73
|
+
if (existingOutput?.isDirectory()) {
|
|
74
|
+
throw new Error("--output must be a file path for xhs download-media.");
|
|
75
|
+
}
|
|
76
|
+
if (existingOutput) {
|
|
77
|
+
return buildSkippedExistingResult(parsedUrl.href, target.outputPath, existingOutput.size);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
return downloadWithPartResume(parsedUrl, target, {
|
|
81
|
+
fetchMedia,
|
|
82
|
+
downloadTimeoutMs,
|
|
83
|
+
maxDownloadBytes,
|
|
84
|
+
retryDelayMs,
|
|
85
|
+
linkFile,
|
|
86
|
+
copyFile,
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
async function resolveXhsMediaTarget(parsedUrl, options = {}, now) {
|
|
91
|
+
if (options.output && options.outputDir) {
|
|
92
|
+
throw new Error("Use only one of --output or --output-dir for xhs download-media.");
|
|
93
|
+
}
|
|
94
|
+
if (options.output) {
|
|
95
|
+
const outputPath = resolve(expandHome(options.output));
|
|
96
|
+
await ensureParentDirectory(outputPath, "--output parent path");
|
|
97
|
+
return {
|
|
98
|
+
outputPath,
|
|
99
|
+
partPath: `${outputPath}.part`,
|
|
100
|
+
fixedOutputPath: true,
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
if (!options.outputDir) {
|
|
104
|
+
throw new Error("Missing --output or --output-dir for xhs download-media.");
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
const outputDir = resolve(expandHome(options.outputDir));
|
|
108
|
+
const existingOutputDir = await stat(outputDir).catch(ignoreMissing);
|
|
109
|
+
if (existingOutputDir && !existingOutputDir.isDirectory()) {
|
|
110
|
+
throw new Error("--output-dir must be a directory for xhs download-media.");
|
|
111
|
+
}
|
|
112
|
+
if (!existingOutputDir) {
|
|
113
|
+
await ensureParentDirectory(outputDir, "--output-dir parent path");
|
|
114
|
+
await mkdir(outputDir, { recursive: true });
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
const fileTarget = inferMediaFileTarget(parsedUrl, now);
|
|
118
|
+
const initialExtension =
|
|
119
|
+
fileTarget.extension || fileTarget.urlFormatExtension || "bin";
|
|
120
|
+
const fileName = formatMediaFileName(fileTarget.stem, initialExtension);
|
|
121
|
+
const outputPath = join(outputDir, fileName);
|
|
122
|
+
return {
|
|
123
|
+
outputPath,
|
|
124
|
+
partPath: `${outputPath}.part`,
|
|
125
|
+
outputDir,
|
|
126
|
+
fileStem: fileTarget.stem,
|
|
127
|
+
extension: fileTarget.extension,
|
|
128
|
+
urlFormatExtension: fileTarget.urlFormatExtension,
|
|
129
|
+
fixedOutputPath: false,
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
async function downloadWithPartResume(
|
|
134
|
+
parsedUrl,
|
|
135
|
+
target,
|
|
136
|
+
{
|
|
137
|
+
fetchMedia,
|
|
138
|
+
downloadTimeoutMs,
|
|
139
|
+
maxDownloadBytes,
|
|
140
|
+
retryDelayMs,
|
|
141
|
+
linkFile,
|
|
142
|
+
copyFile,
|
|
143
|
+
}
|
|
144
|
+
) {
|
|
145
|
+
let lastError;
|
|
146
|
+
for (let attempt = 1; attempt <= XHS_DOWNLOAD_MAX_ATTEMPTS; attempt += 1) {
|
|
147
|
+
try {
|
|
148
|
+
const result = await downloadOnce(parsedUrl, target, {
|
|
149
|
+
fetchMedia,
|
|
150
|
+
downloadTimeoutMs,
|
|
151
|
+
maxDownloadBytes,
|
|
152
|
+
linkFile,
|
|
153
|
+
copyFile,
|
|
154
|
+
});
|
|
155
|
+
if (result.skippedExisting) {
|
|
156
|
+
return buildSkippedExistingResult(
|
|
157
|
+
parsedUrl.href,
|
|
158
|
+
result.outputPath,
|
|
159
|
+
result.outputBytes
|
|
160
|
+
);
|
|
161
|
+
}
|
|
162
|
+
return {
|
|
163
|
+
platform: "xhs",
|
|
164
|
+
action: "download-media",
|
|
165
|
+
status: "downloaded",
|
|
166
|
+
url: parsedUrl.href,
|
|
167
|
+
output_path: result.outputPath,
|
|
168
|
+
output_bytes: result.outputBytes,
|
|
169
|
+
resumed: result.resumed,
|
|
170
|
+
content_type: result.contentType,
|
|
171
|
+
};
|
|
172
|
+
} catch (error) {
|
|
173
|
+
if (error instanceof XhsExpiredMediaLinkError) {
|
|
174
|
+
throw error;
|
|
175
|
+
}
|
|
176
|
+
lastError = error;
|
|
177
|
+
if (!shouldRetryDownloadError(error)) {
|
|
178
|
+
throw error;
|
|
179
|
+
}
|
|
180
|
+
if (attempt === XHS_DOWNLOAD_MAX_ATTEMPTS) {
|
|
181
|
+
break;
|
|
182
|
+
}
|
|
183
|
+
await sleep(retryDelayMs);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
if (isAbortLikeError(lastError)) {
|
|
188
|
+
throw new Error("XHS media download timed out.");
|
|
189
|
+
}
|
|
190
|
+
throw new Error("XHS media download failed before completion.");
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
async function downloadOnce(
|
|
194
|
+
parsedUrl,
|
|
195
|
+
target,
|
|
196
|
+
{
|
|
197
|
+
fetchMedia,
|
|
198
|
+
downloadTimeoutMs,
|
|
199
|
+
maxDownloadBytes,
|
|
200
|
+
linkFile,
|
|
201
|
+
copyFile,
|
|
202
|
+
}
|
|
203
|
+
) {
|
|
204
|
+
await mkdir(dirname(target.outputPath), { recursive: true });
|
|
205
|
+
const partPath = target.partPath;
|
|
206
|
+
const existingPart = await stat(partPath).catch(ignoreMissing);
|
|
207
|
+
let existingPartBytes = existingPart?.isFile() ? existingPart.size : 0;
|
|
208
|
+
const headers = normalizeXhsDownloadHeaders();
|
|
209
|
+
if (existingPartBytes > 0) {
|
|
210
|
+
headers.Range = `bytes=${existingPartBytes}-`;
|
|
211
|
+
delete headers["accept-encoding"];
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
const signal = createMediaDownloadAbortSignal(downloadTimeoutMs);
|
|
215
|
+
const response = await fetchMedia(parsedUrl.href, {
|
|
216
|
+
headers,
|
|
217
|
+
...(signal ? { signal } : {}),
|
|
218
|
+
});
|
|
219
|
+
|
|
220
|
+
if (XHS_EXPIRED_HTTP_STATUSES.has(response.status)) {
|
|
221
|
+
throw new XhsExpiredMediaLinkError();
|
|
222
|
+
}
|
|
223
|
+
if (response.status === 416) {
|
|
224
|
+
return handleRangeNotSatisfiable(response, {
|
|
225
|
+
target,
|
|
226
|
+
partPath,
|
|
227
|
+
existingPartBytes,
|
|
228
|
+
linkFile,
|
|
229
|
+
copyFile,
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
if (!response.ok) {
|
|
233
|
+
if (XHS_RETRYABLE_HTTP_STATUSES.has(response.status)) {
|
|
234
|
+
throw new XhsRetryableDownloadError();
|
|
235
|
+
}
|
|
236
|
+
throw new Error(`XHS media download failed with HTTP ${response.status}.`);
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
const contentType = normalizedContentType(headerValue(response.headers, "content-type"));
|
|
240
|
+
const outputPath = resolveFinalOutputPath(target, contentType);
|
|
241
|
+
const existingOutput = await stat(outputPath).catch(ignoreMissing);
|
|
242
|
+
if (existingOutput?.isDirectory()) {
|
|
243
|
+
throw new Error("--output must be a file path for xhs download-media.");
|
|
244
|
+
}
|
|
245
|
+
if (existingOutput) {
|
|
246
|
+
await cancelResponseBody(response);
|
|
247
|
+
return {
|
|
248
|
+
skippedExisting: true,
|
|
249
|
+
outputPath,
|
|
250
|
+
outputBytes: existingOutput.size,
|
|
251
|
+
contentType,
|
|
252
|
+
};
|
|
253
|
+
}
|
|
254
|
+
const contentLength = parseOptionalIntegerHeader(
|
|
255
|
+
headerValue(response.headers, "content-length")
|
|
256
|
+
);
|
|
257
|
+
let append = false;
|
|
258
|
+
let expectedTotalBytes = contentLength;
|
|
259
|
+
let expectedResponseBytes;
|
|
260
|
+
|
|
261
|
+
if (existingPartBytes > 0 && response.status === 206) {
|
|
262
|
+
const contentRange = parseContentRange(
|
|
263
|
+
headerValue(response.headers, "content-range")
|
|
264
|
+
);
|
|
265
|
+
if (
|
|
266
|
+
!isUsableContentRange(contentRange, contentLength) ||
|
|
267
|
+
contentRange.start !== existingPartBytes
|
|
268
|
+
) {
|
|
269
|
+
await cancelResponseBody(response);
|
|
270
|
+
await unlink(partPath).catch(ignoreError);
|
|
271
|
+
throw new XhsRetryableDownloadError();
|
|
272
|
+
}
|
|
273
|
+
append = true;
|
|
274
|
+
expectedResponseBytes = contentRange.end - contentRange.start + 1;
|
|
275
|
+
expectedTotalBytes = contentRange.total;
|
|
276
|
+
} else if (existingPartBytes > 0) {
|
|
277
|
+
existingPartBytes = 0;
|
|
278
|
+
expectedTotalBytes = contentLength;
|
|
279
|
+
} else if (response.status === 206) {
|
|
280
|
+
const contentRange = parseContentRange(
|
|
281
|
+
headerValue(response.headers, "content-range")
|
|
282
|
+
);
|
|
283
|
+
if (
|
|
284
|
+
!isUsableContentRange(contentRange, contentLength) ||
|
|
285
|
+
contentRange.start !== 0
|
|
286
|
+
) {
|
|
287
|
+
await cancelResponseBody(response);
|
|
288
|
+
throw new XhsRetryableDownloadError();
|
|
289
|
+
}
|
|
290
|
+
expectedResponseBytes = contentRange.end - contentRange.start + 1;
|
|
291
|
+
expectedTotalBytes = contentRange.total;
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
assertMediaDownloadWithinLimit(expectedTotalBytes, maxDownloadBytes);
|
|
295
|
+
let outputBytes;
|
|
296
|
+
let bytesWritten;
|
|
297
|
+
try {
|
|
298
|
+
bytesWritten = await writeResponseBodyToPart(response, partPath, {
|
|
299
|
+
append,
|
|
300
|
+
existingBytes: existingPartBytes,
|
|
301
|
+
maxBytes: maxDownloadBytes,
|
|
302
|
+
});
|
|
303
|
+
outputBytes = existingPartBytes + bytesWritten;
|
|
304
|
+
} catch (error) {
|
|
305
|
+
if (isAbortLikeError(error) || isDownloadTooLargeError(error)) {
|
|
306
|
+
throw error;
|
|
307
|
+
}
|
|
308
|
+
throw new XhsRetryableDownloadError();
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
if (
|
|
312
|
+
expectedResponseBytes !== undefined &&
|
|
313
|
+
bytesWritten !== expectedResponseBytes
|
|
314
|
+
) {
|
|
315
|
+
if (bytesWritten > expectedResponseBytes) {
|
|
316
|
+
await unlink(partPath).catch(ignoreError);
|
|
317
|
+
}
|
|
318
|
+
throw new XhsRetryableDownloadError();
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
if (expectedTotalBytes !== undefined && outputBytes !== expectedTotalBytes) {
|
|
322
|
+
if (outputBytes > expectedTotalBytes) {
|
|
323
|
+
await unlink(partPath).catch(ignoreError);
|
|
324
|
+
}
|
|
325
|
+
throw new XhsRetryableDownloadError();
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
await finalizePartFile(partPath, outputPath, { linkFile, copyFile });
|
|
329
|
+
return {
|
|
330
|
+
outputPath,
|
|
331
|
+
outputBytes,
|
|
332
|
+
resumed: append && existingPartBytes > 0,
|
|
333
|
+
contentType,
|
|
334
|
+
};
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
async function handleRangeNotSatisfiable(
|
|
338
|
+
response,
|
|
339
|
+
{
|
|
340
|
+
target,
|
|
341
|
+
partPath,
|
|
342
|
+
existingPartBytes,
|
|
343
|
+
linkFile,
|
|
344
|
+
copyFile,
|
|
345
|
+
}
|
|
346
|
+
) {
|
|
347
|
+
const contentRange = parseContentRange(headerValue(response.headers, "content-range"));
|
|
348
|
+
if (
|
|
349
|
+
existingPartBytes > 0 &&
|
|
350
|
+
contentRange?.total !== undefined &&
|
|
351
|
+
existingPartBytes === contentRange.total
|
|
352
|
+
) {
|
|
353
|
+
const contentType = normalizedContentType(headerValue(response.headers, "content-type"));
|
|
354
|
+
const outputPath = resolveFinalOutputPath(target, contentType);
|
|
355
|
+
const existingOutput = await stat(outputPath).catch(ignoreMissing);
|
|
356
|
+
if (existingOutput?.isDirectory()) {
|
|
357
|
+
throw new Error("--output must be a file path for xhs download-media.");
|
|
358
|
+
}
|
|
359
|
+
if (existingOutput) {
|
|
360
|
+
await cancelResponseBody(response);
|
|
361
|
+
return {
|
|
362
|
+
skippedExisting: true,
|
|
363
|
+
outputPath,
|
|
364
|
+
outputBytes: existingOutput.size,
|
|
365
|
+
contentType,
|
|
366
|
+
};
|
|
367
|
+
}
|
|
368
|
+
await finalizePartFile(partPath, outputPath, { linkFile, copyFile });
|
|
369
|
+
return {
|
|
370
|
+
outputPath,
|
|
371
|
+
outputBytes: existingPartBytes,
|
|
372
|
+
resumed: true,
|
|
373
|
+
contentType,
|
|
374
|
+
};
|
|
375
|
+
}
|
|
376
|
+
await unlink(partPath).catch(ignoreError);
|
|
377
|
+
throw new XhsRetryableDownloadError();
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
async function writeResponseBodyToPart(
|
|
381
|
+
response,
|
|
382
|
+
partPath,
|
|
383
|
+
{
|
|
384
|
+
append,
|
|
385
|
+
existingBytes,
|
|
386
|
+
maxBytes,
|
|
387
|
+
}
|
|
388
|
+
) {
|
|
389
|
+
if (!response.body) {
|
|
390
|
+
throw new XhsRetryableDownloadError();
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
const file = await openFile(partPath, append ? "r+" : "w");
|
|
394
|
+
let bytesWritten = 0;
|
|
395
|
+
try {
|
|
396
|
+
for await (const chunk of response.body) {
|
|
397
|
+
const buffer = Buffer.from(chunk);
|
|
398
|
+
if (buffer.length === 0) {
|
|
399
|
+
continue;
|
|
400
|
+
}
|
|
401
|
+
const nextTotal = existingBytes + bytesWritten + buffer.length;
|
|
402
|
+
assertMediaDownloadWithinLimit(nextTotal, maxBytes);
|
|
403
|
+
await writeAllToFile(file, buffer, existingBytes + bytesWritten);
|
|
404
|
+
bytesWritten += buffer.length;
|
|
405
|
+
}
|
|
406
|
+
} finally {
|
|
407
|
+
await file.close();
|
|
408
|
+
}
|
|
409
|
+
return bytesWritten;
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
async function finalizePartFile(
|
|
413
|
+
partPath,
|
|
414
|
+
outputPath,
|
|
415
|
+
{
|
|
416
|
+
linkFile = linkPath,
|
|
417
|
+
copyFile = copyFilePath,
|
|
418
|
+
} = {}
|
|
419
|
+
) {
|
|
420
|
+
try {
|
|
421
|
+
await ensureParentDirectory(outputPath, "xhs download-media output parent path");
|
|
422
|
+
try {
|
|
423
|
+
await linkFile(partPath, outputPath);
|
|
424
|
+
} catch (error) {
|
|
425
|
+
if (isExistingOutputPathError(error)) {
|
|
426
|
+
throw error;
|
|
427
|
+
}
|
|
428
|
+
if (!shouldFallbackToCopyAfterLinkError(error)) {
|
|
429
|
+
throw error;
|
|
430
|
+
}
|
|
431
|
+
await copyFile(partPath, outputPath, fsConstants.COPYFILE_EXCL);
|
|
432
|
+
}
|
|
433
|
+
} catch (error) {
|
|
434
|
+
if (isExistingOutputPathError(error)) {
|
|
435
|
+
throw await buildExistingOutputPathError(outputPath);
|
|
436
|
+
}
|
|
437
|
+
throw error;
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
await unlink(partPath);
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
async function cancelResponseBody(response) {
|
|
444
|
+
const body = response.body;
|
|
445
|
+
if (!body) {
|
|
446
|
+
return;
|
|
447
|
+
}
|
|
448
|
+
if (typeof body.cancel === "function") {
|
|
449
|
+
await body.cancel().catch(ignoreError);
|
|
450
|
+
return;
|
|
451
|
+
}
|
|
452
|
+
if (typeof body.return === "function") {
|
|
453
|
+
await body.return().catch(ignoreError);
|
|
454
|
+
return;
|
|
455
|
+
}
|
|
456
|
+
if (typeof body.destroy === "function") {
|
|
457
|
+
body.destroy();
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
function parseHttpUrl(value) {
|
|
462
|
+
let parsed;
|
|
463
|
+
try {
|
|
464
|
+
parsed = new URL(String(value || ""));
|
|
465
|
+
} catch {
|
|
466
|
+
throw new Error("XHS media URL must be an HTTP URL.");
|
|
467
|
+
}
|
|
468
|
+
if (!["http:", "https:"].includes(parsed.protocol) || !parsed.hostname) {
|
|
469
|
+
throw new Error("XHS media URL must be an HTTP URL.");
|
|
470
|
+
}
|
|
471
|
+
return parsed;
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
function normalizeXhsDownloadHeaders() {
|
|
475
|
+
return {
|
|
476
|
+
"User-Agent": XHS_DOWNLOAD_USER_AGENT,
|
|
477
|
+
Referer: XHS_DOWNLOAD_REFERER,
|
|
478
|
+
"accept-encoding": "identity",
|
|
479
|
+
};
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
function resolveFinalOutputPath(target, contentType) {
|
|
483
|
+
if (target.fixedOutputPath) {
|
|
484
|
+
return target.outputPath;
|
|
485
|
+
}
|
|
486
|
+
const extension =
|
|
487
|
+
target.extension ||
|
|
488
|
+
mediaExtensionFromContentType(contentType) ||
|
|
489
|
+
target.urlFormatExtension ||
|
|
490
|
+
"bin";
|
|
491
|
+
return join(target.outputDir, formatMediaFileName(target.fileStem, extension));
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
function inferMediaFileTarget(parsedUrl, now) {
|
|
495
|
+
const fallbackStem = `xhs-media-${formatTimestamp(now())}`;
|
|
496
|
+
const urlFormatExtension = inferUrlFormatExtension(parsedUrl);
|
|
497
|
+
const rawBasename = safeDecodeURIComponent(basename(parsedUrl.pathname || ""));
|
|
498
|
+
const sanitized = sanitizeFileName(rawBasename);
|
|
499
|
+
if (sanitized && sanitized !== "." && sanitized !== "..") {
|
|
500
|
+
const currentExt = extname(sanitized);
|
|
501
|
+
const extension = sanitizeFileExtension(currentExt.slice(1));
|
|
502
|
+
const stem = extension
|
|
503
|
+
? sanitizeFileName(sanitized.slice(0, -currentExt.length)) || fallbackStem
|
|
504
|
+
: sanitized;
|
|
505
|
+
return {
|
|
506
|
+
stem,
|
|
507
|
+
extension,
|
|
508
|
+
urlFormatExtension,
|
|
509
|
+
};
|
|
510
|
+
}
|
|
511
|
+
return {
|
|
512
|
+
stem: fallbackStem,
|
|
513
|
+
extension: "",
|
|
514
|
+
urlFormatExtension,
|
|
515
|
+
};
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
async function ensureParentDirectory(outputPath, displayName) {
|
|
519
|
+
const parent = dirname(outputPath);
|
|
520
|
+
const existingParent = await stat(parent).catch(ignoreMissing);
|
|
521
|
+
if (existingParent && !existingParent.isDirectory()) {
|
|
522
|
+
throw new Error(`${displayName} must be a directory for xhs download-media.`);
|
|
523
|
+
}
|
|
524
|
+
if (!existingParent) {
|
|
525
|
+
await mkdir(parent, { recursive: true });
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
function buildSkippedExistingResult(url, outputPath, outputBytes) {
|
|
530
|
+
return {
|
|
531
|
+
platform: "xhs",
|
|
532
|
+
action: "download-media",
|
|
533
|
+
status: "skipped_existing",
|
|
534
|
+
url,
|
|
535
|
+
output_path: outputPath,
|
|
536
|
+
output_bytes: outputBytes,
|
|
537
|
+
};
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
function parseContentRange(value) {
|
|
541
|
+
if (!value) {
|
|
542
|
+
return undefined;
|
|
543
|
+
}
|
|
544
|
+
const trimmed = String(value).trim();
|
|
545
|
+
const unsatisfied = /^bytes\s+\*\/(\d+)$/i.exec(trimmed);
|
|
546
|
+
if (unsatisfied) {
|
|
547
|
+
return {
|
|
548
|
+
start: undefined,
|
|
549
|
+
end: undefined,
|
|
550
|
+
total: Number.parseInt(unsatisfied[1], 10),
|
|
551
|
+
};
|
|
552
|
+
}
|
|
553
|
+
const matched = /^bytes\s+(\d+)-(\d+)\/(\d+|\*)$/i.exec(trimmed);
|
|
554
|
+
if (!matched) {
|
|
555
|
+
return undefined;
|
|
556
|
+
}
|
|
557
|
+
return {
|
|
558
|
+
start: Number.parseInt(matched[1], 10),
|
|
559
|
+
end: Number.parseInt(matched[2], 10),
|
|
560
|
+
total: matched[3] === "*" ? undefined : Number.parseInt(matched[3], 10),
|
|
561
|
+
};
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
function isUsableContentRange(contentRange, contentLength) {
|
|
565
|
+
if (
|
|
566
|
+
contentRange?.start === undefined ||
|
|
567
|
+
contentRange.end === undefined ||
|
|
568
|
+
contentRange.total === undefined ||
|
|
569
|
+
!Number.isSafeInteger(contentRange.start) ||
|
|
570
|
+
!Number.isSafeInteger(contentRange.end) ||
|
|
571
|
+
!Number.isSafeInteger(contentRange.total) ||
|
|
572
|
+
contentRange.end < contentRange.start ||
|
|
573
|
+
contentRange.end >= contentRange.total
|
|
574
|
+
) {
|
|
575
|
+
return false;
|
|
576
|
+
}
|
|
577
|
+
const rangeBytes = contentRange.end - contentRange.start + 1;
|
|
578
|
+
return contentLength === undefined || contentLength === rangeBytes;
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
function shouldRetryDownloadError(error) {
|
|
582
|
+
return (
|
|
583
|
+
error instanceof XhsRetryableDownloadError ||
|
|
584
|
+
isAbortLikeError(error) ||
|
|
585
|
+
isNetworkLikeError(error)
|
|
586
|
+
);
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
function isNetworkLikeError(error) {
|
|
590
|
+
return (
|
|
591
|
+
error?.name === "TypeError" ||
|
|
592
|
+
["ECONNRESET", "ECONNREFUSED", "EPIPE", "ETIMEDOUT"].includes(error?.code)
|
|
593
|
+
);
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
function isDownloadTooLargeError(error) {
|
|
597
|
+
return error?.message === "Media download is too large for local media processing.";
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
function normalizedContentType(value) {
|
|
601
|
+
if (!value) {
|
|
602
|
+
return null;
|
|
603
|
+
}
|
|
604
|
+
return String(value).split(";")[0].trim().toLowerCase() || null;
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
function headerValue(headers, name) {
|
|
608
|
+
if (!headers || typeof headers.get !== "function") {
|
|
609
|
+
return null;
|
|
610
|
+
}
|
|
611
|
+
return headers.get(name);
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
function sanitizeFileName(value) {
|
|
615
|
+
const normalized = String(value || "")
|
|
616
|
+
.replace(/[\\/:*?"<>|\u0000-\u001f]/g, "_")
|
|
617
|
+
.replace(/\s+/g, " ")
|
|
618
|
+
.trim();
|
|
619
|
+
return normalized.slice(0, 120);
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
function sanitizeFileExtension(value) {
|
|
623
|
+
const normalized = String(value || "").toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
624
|
+
if (!normalized || normalized.length > 8) {
|
|
625
|
+
return "";
|
|
626
|
+
}
|
|
627
|
+
return normalized;
|
|
628
|
+
}
|
|
629
|
+
|
|
630
|
+
function formatMediaFileName(stem, extension) {
|
|
631
|
+
return `${stem}.${sanitizeFileExtension(extension) || "bin"}`;
|
|
632
|
+
}
|
|
633
|
+
|
|
634
|
+
function mediaExtensionFromContentType(contentType) {
|
|
635
|
+
return XHS_CONTENT_TYPE_EXTENSIONS.get(contentType) || "";
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
function inferUrlFormatExtension(parsedUrl) {
|
|
639
|
+
const matched = /(?:^|[/?&])format\/([a-z0-9]+)(?:[/?&#]|$)/i.exec(
|
|
640
|
+
`${parsedUrl.pathname}${parsedUrl.search}`
|
|
641
|
+
);
|
|
642
|
+
if (!matched) {
|
|
643
|
+
return "";
|
|
644
|
+
}
|
|
645
|
+
return XHS_URL_FORMAT_EXTENSIONS.get(sanitizeFileExtension(matched[1])) || "";
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
function isExistingOutputPathError(error) {
|
|
649
|
+
return error?.code === "EEXIST" || error?.code === "EISDIR";
|
|
650
|
+
}
|
|
651
|
+
|
|
652
|
+
function shouldFallbackToCopyAfterLinkError(error) {
|
|
653
|
+
return ["EPERM", "ENOTSUP", "ENOSYS", "EOPNOTSUPP", "EXDEV"].includes(
|
|
654
|
+
error?.code
|
|
655
|
+
);
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
async function buildExistingOutputPathError(outputPath) {
|
|
659
|
+
const existingOutput = await stat(outputPath).catch(ignoreMissing);
|
|
660
|
+
if (existingOutput?.isDirectory()) {
|
|
661
|
+
return new Error("--output must be a file path for xhs download-media.");
|
|
662
|
+
}
|
|
663
|
+
return new Error("XHS media download output file already exists.");
|
|
664
|
+
}
|
|
665
|
+
|
|
666
|
+
function formatTimestamp(value) {
|
|
667
|
+
const date = value instanceof Date ? value : new Date(value);
|
|
668
|
+
const pad = (part) => String(part).padStart(2, "0");
|
|
669
|
+
return [
|
|
670
|
+
date.getFullYear(),
|
|
671
|
+
pad(date.getMonth() + 1),
|
|
672
|
+
pad(date.getDate()),
|
|
673
|
+
pad(date.getHours()),
|
|
674
|
+
pad(date.getMinutes()),
|
|
675
|
+
pad(date.getSeconds()),
|
|
676
|
+
].join("");
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
function safeDecodeURIComponent(value) {
|
|
680
|
+
try {
|
|
681
|
+
return decodeURIComponent(value);
|
|
682
|
+
} catch {
|
|
683
|
+
return value;
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
function expandHome(path) {
|
|
688
|
+
if (!path || path === "~") {
|
|
689
|
+
return homedir();
|
|
690
|
+
}
|
|
691
|
+
if (path.startsWith("~/")) {
|
|
692
|
+
return join(homedir(), path.slice(2));
|
|
693
|
+
}
|
|
694
|
+
return path;
|
|
695
|
+
}
|
|
696
|
+
|
|
697
|
+
function ignoreMissing(error) {
|
|
698
|
+
if (error?.code === "ENOENT" || error?.code === "ENOTDIR") {
|
|
699
|
+
return undefined;
|
|
700
|
+
}
|
|
701
|
+
throw error;
|
|
702
|
+
}
|
|
703
|
+
|
|
704
|
+
function ignoreError() {}
|
|
705
|
+
|
|
706
|
+
function sleep(ms) {
|
|
707
|
+
if (!Number.isFinite(ms) || ms <= 0) {
|
|
708
|
+
return Promise.resolve();
|
|
709
|
+
}
|
|
710
|
+
return new Promise((resolvePromise) => {
|
|
711
|
+
setTimeout(resolvePromise, ms);
|
|
712
|
+
});
|
|
713
|
+
}
|
|
714
|
+
|
|
715
|
+
class XhsRetryableDownloadError extends Error {}
|
|
716
|
+
|
|
717
|
+
class XhsExpiredMediaLinkError extends Error {
|
|
718
|
+
constructor() {
|
|
719
|
+
super(
|
|
720
|
+
"XHS media link is unavailable or expired. Get a fresh media URL from the XHS detail result and retry."
|
|
721
|
+
);
|
|
722
|
+
}
|
|
723
|
+
}
|
package/package.json
CHANGED