socialdatax-skills 0.2.25 → 0.2.27

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -2,7 +2,7 @@
2
2
 
3
3
  This public package provides the unified skill installer and direct CLI helper for SocialDataX services.
4
4
 
5
- The current public tools support 小红书 / Xiaohongshu / XHS / RedNote, 抖音 / Douyin, 快手 / Kuaishou / Kwai, 微博 / Weibo, 视频号 / WeChat Channels, and 微信公众号 / WeChat Official Account article content research and analysis workflows, plus Bilibili / 哔哩哔哩 / B站 local video download and 敏感词检测 / 违禁词检查 text checks. The public skill layer is intentionally named by capability so supported services can evolve without changing the installation model.
5
+ The current public tools support 小红书 / Xiaohongshu / XHS / RedNote, 抖音 / Douyin, 快手 / Kuaishou / Kwai, 微博 / Weibo, 视频号 / WeChat Channels, and 微信公众号 / WeChat Official Account article content research and analysis workflows, plus XHS local media download, Bilibili / 哔哩哔哩 / B站 local video download, and 敏感词检测 / 违禁词检查 text checks. The public skill layer is intentionally named by capability so supported services can evolve without changing the installation model.
6
6
 
7
7
  - direct `npx` JSON commands for agents that can run shell commands
8
8
  - AgentSkills-compatible installers split by capability for OpenClaw, Hermes Agent, Codex, Claude Code, and general agent skill directories
@@ -28,6 +28,9 @@ Common search phrases for this skill package:
28
28
  - `Xiaohongshu skills`
29
29
  - `XHS skills`
30
30
  - `RedNote skills`
31
+ - `XHS media download`
32
+ - `小红书图片下载`
33
+ - `小红书视频下载`
31
34
  - `Douyin skills`
32
35
  - `抖音 skills`
33
36
  - `Kuaishou skills`
@@ -79,7 +82,7 @@ Common search phrases for this skill package:
79
82
  - Sensitive Words Check MCP server name: `sensitive-check`
80
83
  - Sensitive Words Check future MCP registry name: `com.socialdatax/sensitive-check`
81
84
  - Unified MCP registry name: none; this package installs skills and calls explicit platform services.
82
- - Current public capability version: `0.2.25`
85
+ - Current public capability version: `0.2.27`
83
86
 
84
87
  ## Direct CLI
85
88
 
@@ -102,6 +105,7 @@ npx -y socialdatax-skills@latest xhs user-posts --user-id "<user_id>" --since-da
102
105
  npx -y socialdatax-skills@latest xhs transcript --url "<note_url_or_share_text>" --pretty
103
106
  npx -y socialdatax-skills@latest xhs transcript --note-id "<note_id>" --pretty
104
107
  npx -y socialdatax-skills@latest xhs transcript --job-id "<job_id>" --pretty
108
+ npx -y socialdatax-skills@latest xhs download-media --url "<xhs_media_url>" --output-dir ./downloads --pretty
105
109
  npx -y socialdatax-skills@latest douyin hot-search --pretty
106
110
  npx -y socialdatax-skills@latest douyin search --keyword "露营" --pretty
107
111
  npx -y socialdatax-skills@latest douyin detail --aweme-id "<aweme_id>" --pretty
@@ -170,7 +174,7 @@ npx -y socialdatax-skills@latest wechat transcript --job-id "<job_id>" --pretty
170
174
  npx -y socialdatax-skills@latest sensitive-check text --text "<content>" --platform xhs --pretty
171
175
  ```
172
176
 
173
- Most direct CLI commands print a JSON envelope with `platform`, `tool`, `arguments`, and `data`. `sensitive-check` prints `platform`, `tool`, and `data` only; it does not echo the original text in CLI output. `wechat decrypt-media` is a local save command: pass the `video.video_url` returned by WeChat detail and an `--output` file path. It saves the media locally, decrypts when needed, and does not require `SOCIALDATAX_API_KEY`.
177
+ Most direct CLI commands print a JSON envelope with `platform`, `tool`, `arguments`, and `data`. `sensitive-check` prints `platform`, `tool`, and `data` only; it does not echo the original text in CLI output. `wechat decrypt-media` is a local save command: pass the `video.video_url` returned by WeChat detail and an `--output` file path. It saves the media locally, decrypts when needed, and does not require `SOCIALDATAX_API_KEY`. `xhs download-media` is also local: pass one image or video media URL returned by XHS detail and either `--output <file>` or `--output-dir <directory>`. It writes through a `.part` file, resumes partial downloads when the server supports range requests, skips an already existing output file, and infers common image/video extensions in `--output-dir` mode.
174
178
 
175
179
  `bilibili download` calls SocialDataX once to fetch short-lived download links, then downloads video and audio tracks on the local machine and merges them with local `ffmpeg`. Set `SOCIALDATAX_API_KEY`, install `ffmpeg`, and pass either `--output <file>` or `--output-dir <directory>`. The download-links request consumes 10 credits; the local track download and merge do not consume additional SocialDataX credits. Use `--ffmpeg-path <path>` when `ffmpeg` is not on `PATH`, and `--keep-tracks` to retain the separate track files after merge.
176
180
 
package/cli.mjs CHANGED
@@ -12,13 +12,17 @@ import { homedir } from "node:os";
12
12
  import { dirname, join, resolve } from "node:path";
13
13
  import { fileURLToPath } from "node:url";
14
14
 
15
- import { downloadBilibiliVideoFromManifest } from "./lib/media/bilibili-download.mjs";
15
+ import {
16
+ assertBilibiliFfmpegAvailable,
17
+ downloadBilibiliVideoFromManifest,
18
+ } from "./lib/media/bilibili-download.mjs";
16
19
  import { decryptWechatMediaCommand } from "./lib/media/wechat-decrypt.mjs";
20
+ import { downloadXhsMediaFromUrl } from "./lib/media/xhs-download.mjs";
17
21
 
18
22
  export { decryptWechatMediaCommand };
19
23
 
20
24
  const PACKAGE_NAME = "socialdatax-skills";
21
- const PACKAGE_VERSION = "0.2.25";
25
+ const PACKAGE_VERSION = "0.2.27";
22
26
  const PACKAGE_SPEC = `${PACKAGE_NAME}@latest`;
23
27
  const LOG_PREFIX = `[${PACKAGE_NAME}]`;
24
28
  const MIN_NODE_VERSION = "20.18.1";
@@ -160,6 +164,7 @@ const XHS_DIRECT_ACTION_OPTIONS = {
160
164
  "pretty",
161
165
  ],
162
166
  transcript: ["url", "noteId", "jobId", "pretty"],
167
+ "download-media": ["url", "output", "outputDir", "pretty"],
163
168
  };
164
169
  const XHS_DIRECT_ACTION_NAMES = Object.keys(XHS_DIRECT_ACTION_OPTIONS).join(", ");
165
170
  const XHS_OPTION_DISPLAY_NAMES = {
@@ -179,6 +184,8 @@ const XHS_OPTION_DISPLAY_NAMES = {
179
184
  profileUrl: "--profile-url",
180
185
  userId: "--user-id",
181
186
  jobId: "--job-id",
187
+ output: "--output",
188
+ outputDir: "--output-dir",
182
189
  };
183
190
  const XHS_SEARCH_SORT_TYPES = [
184
191
  "general",
@@ -527,12 +534,12 @@ const PLATFORMS = {
527
534
  {
528
535
  name: "xhs_get_note_comments_by_note_id",
529
536
  description:
530
- "Fetch paginated first-level comments by note ID with optional sort.",
537
+ "Fetch paginated first-level comments by note ID.",
531
538
  },
532
539
  {
533
540
  name: "xhs_get_note_comments_by_note_url",
534
541
  description:
535
- "Fetch paginated first-level comments from a note URL, short link, or share text with optional sort.",
542
+ "Fetch paginated first-level comments from a note URL, short link, or share text.",
536
543
  },
537
544
  {
538
545
  name: "xhs_get_note_sub_comments_by_comment_id",
@@ -1094,6 +1101,26 @@ function validateXhsDirectActionOptions(action, options) {
1094
1101
  return;
1095
1102
  }
1096
1103
 
1104
+ if (action === "download-media") {
1105
+ validateKnownOptions(options, allowedOptions);
1106
+ validateFlagOption(options, "pretty", "--pretty");
1107
+ for (const key of allowedOptions) {
1108
+ if (!DIRECT_BOOLEAN_OPTIONS.has(key)) {
1109
+ requireOptionValue(options, key, XHS_OPTION_DISPLAY_NAMES[key]);
1110
+ }
1111
+ }
1112
+ if (!options.url) {
1113
+ throw new Error("Missing --url for xhs download-media.");
1114
+ }
1115
+ if (!options.output && !options.outputDir) {
1116
+ throw new Error("Missing --output or --output-dir for xhs download-media.");
1117
+ }
1118
+ if (options.output && options.outputDir) {
1119
+ throw new Error("Use only one of --output or --output-dir for xhs download-media.");
1120
+ }
1121
+ return;
1122
+ }
1123
+
1097
1124
  validateKnownOptions(options, allowedDirectOptions(allowedOptions));
1098
1125
  validateDirectMetaOptions(options);
1099
1126
  validateDirectPaginationOptions("xhs", action, options);
@@ -1844,6 +1871,9 @@ function printHelp() {
1844
1871
  console.log(` npx -y ${PACKAGE_SPEC} xhs transcript --url "<xhs_note_url_or_share_text>" --pretty`);
1845
1872
  console.log(" Submit or check an XHS video note speech-to-text transcript job.");
1846
1873
  console.log("");
1874
+ console.log(` npx -y ${PACKAGE_SPEC} xhs download-media --url "<xhs_media_url>" --output-dir ./downloads --pretty`);
1875
+ console.log(" Save one XHS image or video media URL returned by detail to a local file.");
1876
+ console.log("");
1847
1877
  console.log(` npx -y ${PACKAGE_SPEC} douyin hot-search --pretty`);
1848
1878
  console.log(" Call the Douyin main hot search list tool directly and print JSON.");
1849
1879
  console.log("");
@@ -2087,13 +2117,14 @@ function printHelp() {
2087
2117
  console.log(" --keyword <text>");
2088
2118
  console.log(" --url <url-or-share-text>");
2089
2119
  console.log(" Content link, short link, or share text for URL-based detail/comment/article commands.");
2120
+ console.log(" For xhs download-media, pass one image or video media URL returned by XHS detail.");
2090
2121
  console.log(" For Douyin detail/comments, pass a content page link, not video.play_url.");
2091
2122
  console.log(" --media-url <video.video_url>");
2092
2123
  console.log(" WeChat detail result media URL for local decrypt-media download.");
2093
2124
  console.log(" --output <file>");
2094
- console.log(" Output file path for local decrypt-media or Bilibili download.");
2125
+ console.log(" Output file path for local decrypt-media, XHS download-media, or Bilibili download.");
2095
2126
  console.log(" --output-dir <directory>");
2096
- console.log(" Directory for Bilibili download output when --output is omitted.");
2127
+ console.log(" Directory for XHS download-media or Bilibili download output when --output is omitted.");
2097
2128
  console.log(" --ffmpeg-path <path>");
2098
2129
  console.log(" ffmpeg executable path for Bilibili download; defaults to ffmpeg.");
2099
2130
  console.log(" --keep-tracks");
@@ -2215,6 +2246,13 @@ async function runXhsDirectCommand(args) {
2215
2246
  }
2216
2247
  validateXhsDirectActionOptions(action, options);
2217
2248
 
2249
+ if (action === "download-media") {
2250
+ const data = await downloadXhsMediaFromUrl(options.url, options);
2251
+ process.stdout.write(JSON.stringify(data, null, options.pretty ? 2 : 0));
2252
+ process.stdout.write("\n");
2253
+ return;
2254
+ }
2255
+
2218
2256
  const operation = attachDirectMetadata(buildXhsOperation(action, options), options);
2219
2257
  const data = shouldUsePaginatedDirectOutput(options)
2220
2258
  ? await callPaginatedDirectOperation(operation, options)
@@ -2359,6 +2397,10 @@ async function assertBilibiliDownloadLocalPreflight(options) {
2359
2397
  await assertBilibiliParentDirectory(outputDir, "--output-dir parent path");
2360
2398
  }
2361
2399
  }
2400
+
2401
+ ensureSupportedNodeVersion();
2402
+ readDirectApiKey(PLATFORMS.bilibili);
2403
+ await assertBilibiliFfmpegAvailable(options.ffmpegPath || "ffmpeg");
2362
2404
  }
2363
2405
 
2364
2406
  async function assertBilibiliParentDirectory(targetPath, displayName) {
@@ -4203,13 +4245,7 @@ function buildRepliesOperationForComment(platform, comment, parentArguments, par
4203
4245
  async function callMcpBackend(operation) {
4204
4246
  ensureSupportedNodeVersion();
4205
4247
  const { platform, tool } = operation;
4206
- const apiKey = readFirstEnv(platform.apiKeyEnv);
4207
- if (!apiKey) {
4208
- throw new Error(
4209
- `Missing API Key. Set ${PRIMARY_API_KEY_ENV} before running direct CLI calls.`
4210
- );
4211
- }
4212
-
4248
+ const apiKey = readDirectApiKey(platform);
4213
4249
  const { Client, StreamableHTTPClientTransport } = await loadMcpSdkModules();
4214
4250
  const upstreamUrl = resolveUpstreamUrl(platform);
4215
4251
  const client = new Client(
@@ -4258,6 +4294,16 @@ async function callMcpBackend(operation) {
4258
4294
  }
4259
4295
  }
4260
4296
 
4297
+ function readDirectApiKey(platform) {
4298
+ const apiKey = readFirstEnv(platform.apiKeyEnv);
4299
+ if (!apiKey) {
4300
+ throw new Error(
4301
+ `Missing API Key. Set ${PRIMARY_API_KEY_ENV} before running direct CLI calls.`
4302
+ );
4303
+ }
4304
+ return apiKey;
4305
+ }
4306
+
4261
4307
  async function loadMcpSdkModules() {
4262
4308
  if (!mcpSdkModules) {
4263
4309
  const [{ Client }, { StreamableHTTPClientTransport }] = await Promise.all([
@@ -8,7 +8,12 @@ import {
8
8
  stat,
9
9
  unlink,
10
10
  } from "node:fs/promises";
11
- import { homedir } from "node:os";
11
+ import {
12
+ arch as osArch,
13
+ homedir,
14
+ platform as osPlatform,
15
+ release as osRelease,
16
+ } from "node:os";
12
17
  import { dirname, join, resolve } from "node:path";
13
18
 
14
19
  import {
@@ -39,11 +44,18 @@ export async function downloadBilibiliVideoFromManifest(
39
44
  validateBilibiliDownloadManifest(manifest);
40
45
 
41
46
  const outputPath = resolveBilibiliOutputPath(manifest, options);
47
+ const ffmpegPath = options.ffmpegPath || "ffmpeg";
42
48
  await mkdir(dirname(outputPath), { recursive: true });
43
49
  await reserveBilibiliOutputPath(outputPath);
44
50
 
45
51
  const downloadedTracks = [];
46
52
  try {
53
+ await assertBilibiliTrackPathsAvailable(
54
+ manifest.download_manifest.tracks,
55
+ outputPath
56
+ );
57
+ await assertBilibiliFfmpegAvailable(ffmpegPath, spawnProcess);
58
+
47
59
  for (const track of manifest.download_manifest.tracks) {
48
60
  downloadedTracks.push(
49
61
  await downloadBilibiliTrack(track, {
@@ -63,7 +75,7 @@ export async function downloadBilibiliVideoFromManifest(
63
75
  }
64
76
 
65
77
  await mergeBilibiliTracksWithFfmpeg({
66
- ffmpegPath: options.ffmpegPath || "ffmpeg",
78
+ ffmpegPath,
67
79
  videoPath: videoTrack.path,
68
80
  audioPath: audioTrack.path,
69
81
  outputPath,
@@ -222,9 +234,10 @@ async function downloadBilibiliTrack(
222
234
  maxDownloadBytes,
223
235
  }
224
236
  ) {
225
- const trackType = String(track.type);
226
- const ext = sanitizeFileExtension(track.ext || "m4s");
227
- const trackPath = `${outputPath}.${trackType}.${ext}`;
237
+ const { trackType, ext, trackPath } = resolveBilibiliTrackTarget(
238
+ track,
239
+ outputPath
240
+ );
228
241
  const tempPath = `${trackPath}.tmp-${process.pid}-${Date.now()}-${randomUUID()}`;
229
242
  await mkdir(dirname(trackPath), { recursive: true });
230
243
 
@@ -275,6 +288,31 @@ async function downloadBilibiliTrack(
275
288
  };
276
289
  }
277
290
 
291
+ async function assertBilibiliTrackPathsAvailable(tracks, outputPath) {
292
+ for (const track of tracks) {
293
+ const { trackType, trackPath } = resolveBilibiliTrackTarget(track, outputPath);
294
+ const existingTrack = await stat(trackPath).catch((error) => {
295
+ if (error?.code === "ENOENT") {
296
+ return undefined;
297
+ }
298
+ throw error;
299
+ });
300
+ if (existingTrack) {
301
+ throw new Error(`Bilibili ${trackType} track file already exists.`);
302
+ }
303
+ }
304
+ }
305
+
306
+ function resolveBilibiliTrackTarget(track, outputPath) {
307
+ const trackType = String(track.type);
308
+ const ext = sanitizeFileExtension(track.ext || "m4s");
309
+ return {
310
+ trackType,
311
+ ext,
312
+ trackPath: `${outputPath}.${trackType}.${ext}`,
313
+ };
314
+ }
315
+
278
316
  async function reserveBilibiliTrackPath(trackPath, trackType) {
279
317
  let file;
280
318
  try {
@@ -341,7 +379,21 @@ async function mergeBilibiliTracksWithFfmpeg({
341
379
  await runFfmpegProcess(ffmpegPath, args, spawnProcess);
342
380
  }
343
381
 
344
- async function runFfmpegProcess(ffmpegPath, args, spawnProcess) {
382
+ export async function assertBilibiliFfmpegAvailable(
383
+ ffmpegPath = "ffmpeg",
384
+ spawnProcess = spawn
385
+ ) {
386
+ await runFfmpegProcess(ffmpegPath, ["-version"], spawnProcess, {
387
+ failurePrefix: "ffmpeg availability check failed",
388
+ });
389
+ }
390
+
391
+ async function runFfmpegProcess(
392
+ ffmpegPath,
393
+ args,
394
+ spawnProcess,
395
+ { failurePrefix = "ffmpeg failed for bilibili download" } = {}
396
+ ) {
345
397
  return new Promise((resolvePromise, reject) => {
346
398
  let stderr = "";
347
399
  const child = spawnProcess(ffmpegPath, args, {
@@ -358,7 +410,7 @@ async function runFfmpegProcess(ffmpegPath, args, spawnProcess) {
358
410
  if (error?.code === "ENOENT") {
359
411
  reject(
360
412
  new Error(
361
- "ffmpeg not found. Install ffmpeg or pass --ffmpeg-path for bilibili download."
413
+ buildFfmpegMissingMessage(ffmpegPath)
362
414
  )
363
415
  );
364
416
  return;
@@ -372,13 +424,26 @@ async function runFfmpegProcess(ffmpegPath, args, spawnProcess) {
372
424
  }
373
425
  reject(
374
426
  new Error(
375
- `ffmpeg failed for bilibili download with exit code ${status}. ${stderr.trim()}`.trim()
427
+ `${failurePrefix} with exit code ${status}. ${stderr.trim()}`.trim()
376
428
  )
377
429
  );
378
430
  });
379
431
  });
380
432
  }
381
433
 
434
+ function buildFfmpegMissingMessage(ffmpegPath) {
435
+ const detectedSystem = `${osPlatform()} ${osRelease()} ${osArch()}`;
436
+ return [
437
+ `ffmpeg not found at "${ffmpegPath}".`,
438
+ `Detected system: ${detectedSystem}.`,
439
+ "Install ffmpeg, then retry:",
440
+ "macOS (Homebrew): brew install ffmpeg",
441
+ "Ubuntu/Debian: sudo apt update && sudo apt install ffmpeg",
442
+ "Windows (winget): winget install Gyan.FFmpeg",
443
+ "If ffmpeg is already installed, pass --ffmpeg-path /path/to/ffmpeg for bilibili download.",
444
+ ].join("\n");
445
+ }
446
+
382
447
  function stringValue(value) {
383
448
  return typeof value === "string" && value.trim() ? value : undefined;
384
449
  }
@@ -0,0 +1,723 @@
1
+ import { constants as fsConstants } from "node:fs";
2
+ import {
3
+ copyFile as copyFilePath,
4
+ link as linkPath,
5
+ open as openFile,
6
+ mkdir,
7
+ stat,
8
+ unlink,
9
+ } from "node:fs/promises";
10
+ import { homedir } from "node:os";
11
+ import { basename, dirname, extname, join, resolve } from "node:path";
12
+
13
+ import {
14
+ MEDIA_DOWNLOAD_MAX_BYTES,
15
+ MEDIA_DOWNLOAD_TIMEOUT_MS,
16
+ assertMediaDownloadWithinLimit,
17
+ createMediaDownloadAbortSignal,
18
+ isAbortLikeError,
19
+ parseOptionalIntegerHeader,
20
+ writeAllToFile,
21
+ } from "./common.mjs";
22
+
23
+ const XHS_DOWNLOAD_USER_AGENT =
24
+ "Mozilla/5.0 (compatible; SocialDataX/1.0; +https://socialdatax.com)";
25
+ const XHS_DOWNLOAD_REFERER = "https://www.xiaohongshu.com/";
26
+ const XHS_DOWNLOAD_MAX_ATTEMPTS = 10;
27
+ const XHS_DOWNLOAD_RETRY_DELAY_MS = 250;
28
+ const XHS_RETRYABLE_HTTP_STATUSES = new Set([
29
+ 408,
30
+ 409,
31
+ 425,
32
+ 429,
33
+ 500,
34
+ 502,
35
+ 503,
36
+ 504,
37
+ ]);
38
+ const XHS_EXPIRED_HTTP_STATUSES = new Set([403, 404, 410]);
39
+ const XHS_CONTENT_TYPE_EXTENSIONS = new Map([
40
+ ["image/jpeg", "jpg"],
41
+ ["image/jpg", "jpg"],
42
+ ["image/png", "png"],
43
+ ["image/webp", "webp"],
44
+ ["image/heif", "heif"],
45
+ ["image/heic", "heic"],
46
+ ["video/mp4", "mp4"],
47
+ ]);
48
+ const XHS_URL_FORMAT_EXTENSIONS = new Map([
49
+ ["jpg", "jpg"],
50
+ ["jpeg", "jpg"],
51
+ ["png", "png"],
52
+ ["webp", "webp"],
53
+ ["heif", "heif"],
54
+ ["heic", "heic"],
55
+ ["mp4", "mp4"],
56
+ ]);
57
+ export async function downloadXhsMediaFromUrl(
58
+ url,
59
+ options,
60
+ {
61
+ fetchMedia = fetch,
62
+ downloadTimeoutMs = MEDIA_DOWNLOAD_TIMEOUT_MS,
63
+ maxDownloadBytes = MEDIA_DOWNLOAD_MAX_BYTES,
64
+ retryDelayMs = XHS_DOWNLOAD_RETRY_DELAY_MS,
65
+ now = () => new Date(),
66
+ linkFile = linkPath,
67
+ copyFile = copyFilePath,
68
+ } = {}
69
+ ) {
70
+ const parsedUrl = parseHttpUrl(url);
71
+ const target = await resolveXhsMediaTarget(parsedUrl, options, now);
72
+ const existingOutput = await stat(target.outputPath).catch(ignoreMissing);
73
+ if (existingOutput?.isDirectory()) {
74
+ throw new Error("--output must be a file path for xhs download-media.");
75
+ }
76
+ if (existingOutput) {
77
+ return buildSkippedExistingResult(parsedUrl.href, target.outputPath, existingOutput.size);
78
+ }
79
+
80
+ return downloadWithPartResume(parsedUrl, target, {
81
+ fetchMedia,
82
+ downloadTimeoutMs,
83
+ maxDownloadBytes,
84
+ retryDelayMs,
85
+ linkFile,
86
+ copyFile,
87
+ });
88
+ }
89
+
90
+ async function resolveXhsMediaTarget(parsedUrl, options = {}, now) {
91
+ if (options.output && options.outputDir) {
92
+ throw new Error("Use only one of --output or --output-dir for xhs download-media.");
93
+ }
94
+ if (options.output) {
95
+ const outputPath = resolve(expandHome(options.output));
96
+ await ensureParentDirectory(outputPath, "--output parent path");
97
+ return {
98
+ outputPath,
99
+ partPath: `${outputPath}.part`,
100
+ fixedOutputPath: true,
101
+ };
102
+ }
103
+ if (!options.outputDir) {
104
+ throw new Error("Missing --output or --output-dir for xhs download-media.");
105
+ }
106
+
107
+ const outputDir = resolve(expandHome(options.outputDir));
108
+ const existingOutputDir = await stat(outputDir).catch(ignoreMissing);
109
+ if (existingOutputDir && !existingOutputDir.isDirectory()) {
110
+ throw new Error("--output-dir must be a directory for xhs download-media.");
111
+ }
112
+ if (!existingOutputDir) {
113
+ await ensureParentDirectory(outputDir, "--output-dir parent path");
114
+ await mkdir(outputDir, { recursive: true });
115
+ }
116
+
117
+ const fileTarget = inferMediaFileTarget(parsedUrl, now);
118
+ const initialExtension =
119
+ fileTarget.extension || fileTarget.urlFormatExtension || "bin";
120
+ const fileName = formatMediaFileName(fileTarget.stem, initialExtension);
121
+ const outputPath = join(outputDir, fileName);
122
+ return {
123
+ outputPath,
124
+ partPath: `${outputPath}.part`,
125
+ outputDir,
126
+ fileStem: fileTarget.stem,
127
+ extension: fileTarget.extension,
128
+ urlFormatExtension: fileTarget.urlFormatExtension,
129
+ fixedOutputPath: false,
130
+ };
131
+ }
132
+
133
+ async function downloadWithPartResume(
134
+ parsedUrl,
135
+ target,
136
+ {
137
+ fetchMedia,
138
+ downloadTimeoutMs,
139
+ maxDownloadBytes,
140
+ retryDelayMs,
141
+ linkFile,
142
+ copyFile,
143
+ }
144
+ ) {
145
+ let lastError;
146
+ for (let attempt = 1; attempt <= XHS_DOWNLOAD_MAX_ATTEMPTS; attempt += 1) {
147
+ try {
148
+ const result = await downloadOnce(parsedUrl, target, {
149
+ fetchMedia,
150
+ downloadTimeoutMs,
151
+ maxDownloadBytes,
152
+ linkFile,
153
+ copyFile,
154
+ });
155
+ if (result.skippedExisting) {
156
+ return buildSkippedExistingResult(
157
+ parsedUrl.href,
158
+ result.outputPath,
159
+ result.outputBytes
160
+ );
161
+ }
162
+ return {
163
+ platform: "xhs",
164
+ action: "download-media",
165
+ status: "downloaded",
166
+ url: parsedUrl.href,
167
+ output_path: result.outputPath,
168
+ output_bytes: result.outputBytes,
169
+ resumed: result.resumed,
170
+ content_type: result.contentType,
171
+ };
172
+ } catch (error) {
173
+ if (error instanceof XhsExpiredMediaLinkError) {
174
+ throw error;
175
+ }
176
+ lastError = error;
177
+ if (!shouldRetryDownloadError(error)) {
178
+ throw error;
179
+ }
180
+ if (attempt === XHS_DOWNLOAD_MAX_ATTEMPTS) {
181
+ break;
182
+ }
183
+ await sleep(retryDelayMs);
184
+ }
185
+ }
186
+
187
+ if (isAbortLikeError(lastError)) {
188
+ throw new Error("XHS media download timed out.");
189
+ }
190
+ throw new Error("XHS media download failed before completion.");
191
+ }
192
+
193
+ async function downloadOnce(
194
+ parsedUrl,
195
+ target,
196
+ {
197
+ fetchMedia,
198
+ downloadTimeoutMs,
199
+ maxDownloadBytes,
200
+ linkFile,
201
+ copyFile,
202
+ }
203
+ ) {
204
+ await mkdir(dirname(target.outputPath), { recursive: true });
205
+ const partPath = target.partPath;
206
+ const existingPart = await stat(partPath).catch(ignoreMissing);
207
+ let existingPartBytes = existingPart?.isFile() ? existingPart.size : 0;
208
+ const headers = normalizeXhsDownloadHeaders();
209
+ if (existingPartBytes > 0) {
210
+ headers.Range = `bytes=${existingPartBytes}-`;
211
+ delete headers["accept-encoding"];
212
+ }
213
+
214
+ const signal = createMediaDownloadAbortSignal(downloadTimeoutMs);
215
+ const response = await fetchMedia(parsedUrl.href, {
216
+ headers,
217
+ ...(signal ? { signal } : {}),
218
+ });
219
+
220
+ if (XHS_EXPIRED_HTTP_STATUSES.has(response.status)) {
221
+ throw new XhsExpiredMediaLinkError();
222
+ }
223
+ if (response.status === 416) {
224
+ return handleRangeNotSatisfiable(response, {
225
+ target,
226
+ partPath,
227
+ existingPartBytes,
228
+ linkFile,
229
+ copyFile,
230
+ });
231
+ }
232
+ if (!response.ok) {
233
+ if (XHS_RETRYABLE_HTTP_STATUSES.has(response.status)) {
234
+ throw new XhsRetryableDownloadError();
235
+ }
236
+ throw new Error(`XHS media download failed with HTTP ${response.status}.`);
237
+ }
238
+
239
+ const contentType = normalizedContentType(headerValue(response.headers, "content-type"));
240
+ const outputPath = resolveFinalOutputPath(target, contentType);
241
+ const existingOutput = await stat(outputPath).catch(ignoreMissing);
242
+ if (existingOutput?.isDirectory()) {
243
+ throw new Error("--output must be a file path for xhs download-media.");
244
+ }
245
+ if (existingOutput) {
246
+ await cancelResponseBody(response);
247
+ return {
248
+ skippedExisting: true,
249
+ outputPath,
250
+ outputBytes: existingOutput.size,
251
+ contentType,
252
+ };
253
+ }
254
+ const contentLength = parseOptionalIntegerHeader(
255
+ headerValue(response.headers, "content-length")
256
+ );
257
+ let append = false;
258
+ let expectedTotalBytes = contentLength;
259
+ let expectedResponseBytes;
260
+
261
+ if (existingPartBytes > 0 && response.status === 206) {
262
+ const contentRange = parseContentRange(
263
+ headerValue(response.headers, "content-range")
264
+ );
265
+ if (
266
+ !isUsableContentRange(contentRange, contentLength) ||
267
+ contentRange.start !== existingPartBytes
268
+ ) {
269
+ await cancelResponseBody(response);
270
+ await unlink(partPath).catch(ignoreError);
271
+ throw new XhsRetryableDownloadError();
272
+ }
273
+ append = true;
274
+ expectedResponseBytes = contentRange.end - contentRange.start + 1;
275
+ expectedTotalBytes = contentRange.total;
276
+ } else if (existingPartBytes > 0) {
277
+ existingPartBytes = 0;
278
+ expectedTotalBytes = contentLength;
279
+ } else if (response.status === 206) {
280
+ const contentRange = parseContentRange(
281
+ headerValue(response.headers, "content-range")
282
+ );
283
+ if (
284
+ !isUsableContentRange(contentRange, contentLength) ||
285
+ contentRange.start !== 0
286
+ ) {
287
+ await cancelResponseBody(response);
288
+ throw new XhsRetryableDownloadError();
289
+ }
290
+ expectedResponseBytes = contentRange.end - contentRange.start + 1;
291
+ expectedTotalBytes = contentRange.total;
292
+ }
293
+
294
+ assertMediaDownloadWithinLimit(expectedTotalBytes, maxDownloadBytes);
295
+ let outputBytes;
296
+ let bytesWritten;
297
+ try {
298
+ bytesWritten = await writeResponseBodyToPart(response, partPath, {
299
+ append,
300
+ existingBytes: existingPartBytes,
301
+ maxBytes: maxDownloadBytes,
302
+ });
303
+ outputBytes = existingPartBytes + bytesWritten;
304
+ } catch (error) {
305
+ if (isAbortLikeError(error) || isDownloadTooLargeError(error)) {
306
+ throw error;
307
+ }
308
+ throw new XhsRetryableDownloadError();
309
+ }
310
+
311
+ if (
312
+ expectedResponseBytes !== undefined &&
313
+ bytesWritten !== expectedResponseBytes
314
+ ) {
315
+ if (bytesWritten > expectedResponseBytes) {
316
+ await unlink(partPath).catch(ignoreError);
317
+ }
318
+ throw new XhsRetryableDownloadError();
319
+ }
320
+
321
+ if (expectedTotalBytes !== undefined && outputBytes !== expectedTotalBytes) {
322
+ if (outputBytes > expectedTotalBytes) {
323
+ await unlink(partPath).catch(ignoreError);
324
+ }
325
+ throw new XhsRetryableDownloadError();
326
+ }
327
+
328
+ await finalizePartFile(partPath, outputPath, { linkFile, copyFile });
329
+ return {
330
+ outputPath,
331
+ outputBytes,
332
+ resumed: append && existingPartBytes > 0,
333
+ contentType,
334
+ };
335
+ }
336
+
337
+ async function handleRangeNotSatisfiable(
338
+ response,
339
+ {
340
+ target,
341
+ partPath,
342
+ existingPartBytes,
343
+ linkFile,
344
+ copyFile,
345
+ }
346
+ ) {
347
+ const contentRange = parseContentRange(headerValue(response.headers, "content-range"));
348
+ if (
349
+ existingPartBytes > 0 &&
350
+ contentRange?.total !== undefined &&
351
+ existingPartBytes === contentRange.total
352
+ ) {
353
+ const contentType = normalizedContentType(headerValue(response.headers, "content-type"));
354
+ const outputPath = resolveFinalOutputPath(target, contentType);
355
+ const existingOutput = await stat(outputPath).catch(ignoreMissing);
356
+ if (existingOutput?.isDirectory()) {
357
+ throw new Error("--output must be a file path for xhs download-media.");
358
+ }
359
+ if (existingOutput) {
360
+ await cancelResponseBody(response);
361
+ return {
362
+ skippedExisting: true,
363
+ outputPath,
364
+ outputBytes: existingOutput.size,
365
+ contentType,
366
+ };
367
+ }
368
+ await finalizePartFile(partPath, outputPath, { linkFile, copyFile });
369
+ return {
370
+ outputPath,
371
+ outputBytes: existingPartBytes,
372
+ resumed: true,
373
+ contentType,
374
+ };
375
+ }
376
+ await unlink(partPath).catch(ignoreError);
377
+ throw new XhsRetryableDownloadError();
378
+ }
379
+
380
+ async function writeResponseBodyToPart(
381
+ response,
382
+ partPath,
383
+ {
384
+ append,
385
+ existingBytes,
386
+ maxBytes,
387
+ }
388
+ ) {
389
+ if (!response.body) {
390
+ throw new XhsRetryableDownloadError();
391
+ }
392
+
393
+ const file = await openFile(partPath, append ? "r+" : "w");
394
+ let bytesWritten = 0;
395
+ try {
396
+ for await (const chunk of response.body) {
397
+ const buffer = Buffer.from(chunk);
398
+ if (buffer.length === 0) {
399
+ continue;
400
+ }
401
+ const nextTotal = existingBytes + bytesWritten + buffer.length;
402
+ assertMediaDownloadWithinLimit(nextTotal, maxBytes);
403
+ await writeAllToFile(file, buffer, existingBytes + bytesWritten);
404
+ bytesWritten += buffer.length;
405
+ }
406
+ } finally {
407
+ await file.close();
408
+ }
409
+ return bytesWritten;
410
+ }
411
+
412
+ async function finalizePartFile(
413
+ partPath,
414
+ outputPath,
415
+ {
416
+ linkFile = linkPath,
417
+ copyFile = copyFilePath,
418
+ } = {}
419
+ ) {
420
+ try {
421
+ await ensureParentDirectory(outputPath, "xhs download-media output parent path");
422
+ try {
423
+ await linkFile(partPath, outputPath);
424
+ } catch (error) {
425
+ if (isExistingOutputPathError(error)) {
426
+ throw error;
427
+ }
428
+ if (!shouldFallbackToCopyAfterLinkError(error)) {
429
+ throw error;
430
+ }
431
+ await copyFile(partPath, outputPath, fsConstants.COPYFILE_EXCL);
432
+ }
433
+ } catch (error) {
434
+ if (isExistingOutputPathError(error)) {
435
+ throw await buildExistingOutputPathError(outputPath);
436
+ }
437
+ throw error;
438
+ }
439
+
440
+ await unlink(partPath);
441
+ }
442
+
443
+ async function cancelResponseBody(response) {
444
+ const body = response.body;
445
+ if (!body) {
446
+ return;
447
+ }
448
+ if (typeof body.cancel === "function") {
449
+ await body.cancel().catch(ignoreError);
450
+ return;
451
+ }
452
+ if (typeof body.return === "function") {
453
+ await body.return().catch(ignoreError);
454
+ return;
455
+ }
456
+ if (typeof body.destroy === "function") {
457
+ body.destroy();
458
+ }
459
+ }
460
+
461
+ function parseHttpUrl(value) {
462
+ let parsed;
463
+ try {
464
+ parsed = new URL(String(value || ""));
465
+ } catch {
466
+ throw new Error("XHS media URL must be an HTTP URL.");
467
+ }
468
+ if (!["http:", "https:"].includes(parsed.protocol) || !parsed.hostname) {
469
+ throw new Error("XHS media URL must be an HTTP URL.");
470
+ }
471
+ return parsed;
472
+ }
473
+
474
+ function normalizeXhsDownloadHeaders() {
475
+ return {
476
+ "User-Agent": XHS_DOWNLOAD_USER_AGENT,
477
+ Referer: XHS_DOWNLOAD_REFERER,
478
+ "accept-encoding": "identity",
479
+ };
480
+ }
481
+
482
+ function resolveFinalOutputPath(target, contentType) {
483
+ if (target.fixedOutputPath) {
484
+ return target.outputPath;
485
+ }
486
+ const extension =
487
+ target.extension ||
488
+ mediaExtensionFromContentType(contentType) ||
489
+ target.urlFormatExtension ||
490
+ "bin";
491
+ return join(target.outputDir, formatMediaFileName(target.fileStem, extension));
492
+ }
493
+
494
+ function inferMediaFileTarget(parsedUrl, now) {
495
+ const fallbackStem = `xhs-media-${formatTimestamp(now())}`;
496
+ const urlFormatExtension = inferUrlFormatExtension(parsedUrl);
497
+ const rawBasename = safeDecodeURIComponent(basename(parsedUrl.pathname || ""));
498
+ const sanitized = sanitizeFileName(rawBasename);
499
+ if (sanitized && sanitized !== "." && sanitized !== "..") {
500
+ const currentExt = extname(sanitized);
501
+ const extension = sanitizeFileExtension(currentExt.slice(1));
502
+ const stem = extension
503
+ ? sanitizeFileName(sanitized.slice(0, -currentExt.length)) || fallbackStem
504
+ : sanitized;
505
+ return {
506
+ stem,
507
+ extension,
508
+ urlFormatExtension,
509
+ };
510
+ }
511
+ return {
512
+ stem: fallbackStem,
513
+ extension: "",
514
+ urlFormatExtension,
515
+ };
516
+ }
517
+
518
+ async function ensureParentDirectory(outputPath, displayName) {
519
+ const parent = dirname(outputPath);
520
+ const existingParent = await stat(parent).catch(ignoreMissing);
521
+ if (existingParent && !existingParent.isDirectory()) {
522
+ throw new Error(`${displayName} must be a directory for xhs download-media.`);
523
+ }
524
+ if (!existingParent) {
525
+ await mkdir(parent, { recursive: true });
526
+ }
527
+ }
528
+
529
+ function buildSkippedExistingResult(url, outputPath, outputBytes) {
530
+ return {
531
+ platform: "xhs",
532
+ action: "download-media",
533
+ status: "skipped_existing",
534
+ url,
535
+ output_path: outputPath,
536
+ output_bytes: outputBytes,
537
+ };
538
+ }
539
+
540
+ function parseContentRange(value) {
541
+ if (!value) {
542
+ return undefined;
543
+ }
544
+ const trimmed = String(value).trim();
545
+ const unsatisfied = /^bytes\s+\*\/(\d+)$/i.exec(trimmed);
546
+ if (unsatisfied) {
547
+ return {
548
+ start: undefined,
549
+ end: undefined,
550
+ total: Number.parseInt(unsatisfied[1], 10),
551
+ };
552
+ }
553
+ const matched = /^bytes\s+(\d+)-(\d+)\/(\d+|\*)$/i.exec(trimmed);
554
+ if (!matched) {
555
+ return undefined;
556
+ }
557
+ return {
558
+ start: Number.parseInt(matched[1], 10),
559
+ end: Number.parseInt(matched[2], 10),
560
+ total: matched[3] === "*" ? undefined : Number.parseInt(matched[3], 10),
561
+ };
562
+ }
563
+
564
+ function isUsableContentRange(contentRange, contentLength) {
565
+ if (
566
+ contentRange?.start === undefined ||
567
+ contentRange.end === undefined ||
568
+ contentRange.total === undefined ||
569
+ !Number.isSafeInteger(contentRange.start) ||
570
+ !Number.isSafeInteger(contentRange.end) ||
571
+ !Number.isSafeInteger(contentRange.total) ||
572
+ contentRange.end < contentRange.start ||
573
+ contentRange.end >= contentRange.total
574
+ ) {
575
+ return false;
576
+ }
577
+ const rangeBytes = contentRange.end - contentRange.start + 1;
578
+ return contentLength === undefined || contentLength === rangeBytes;
579
+ }
580
+
581
+ function shouldRetryDownloadError(error) {
582
+ return (
583
+ error instanceof XhsRetryableDownloadError ||
584
+ isAbortLikeError(error) ||
585
+ isNetworkLikeError(error)
586
+ );
587
+ }
588
+
589
+ function isNetworkLikeError(error) {
590
+ return (
591
+ error?.name === "TypeError" ||
592
+ ["ECONNRESET", "ECONNREFUSED", "EPIPE", "ETIMEDOUT"].includes(error?.code)
593
+ );
594
+ }
595
+
596
+ function isDownloadTooLargeError(error) {
597
+ return error?.message === "Media download is too large for local media processing.";
598
+ }
599
+
600
+ function normalizedContentType(value) {
601
+ if (!value) {
602
+ return null;
603
+ }
604
+ return String(value).split(";")[0].trim().toLowerCase() || null;
605
+ }
606
+
607
+ function headerValue(headers, name) {
608
+ if (!headers || typeof headers.get !== "function") {
609
+ return null;
610
+ }
611
+ return headers.get(name);
612
+ }
613
+
614
+ function sanitizeFileName(value) {
615
+ const normalized = String(value || "")
616
+ .replace(/[\\/:*?"<>|\u0000-\u001f]/g, "_")
617
+ .replace(/\s+/g, " ")
618
+ .trim();
619
+ return normalized.slice(0, 120);
620
+ }
621
+
622
+ function sanitizeFileExtension(value) {
623
+ const normalized = String(value || "").toLowerCase().replace(/[^a-z0-9]/g, "");
624
+ if (!normalized || normalized.length > 8) {
625
+ return "";
626
+ }
627
+ return normalized;
628
+ }
629
+
630
+ function formatMediaFileName(stem, extension) {
631
+ return `${stem}.${sanitizeFileExtension(extension) || "bin"}`;
632
+ }
633
+
634
+ function mediaExtensionFromContentType(contentType) {
635
+ return XHS_CONTENT_TYPE_EXTENSIONS.get(contentType) || "";
636
+ }
637
+
638
+ function inferUrlFormatExtension(parsedUrl) {
639
+ const matched = /(?:^|[/?&])format\/([a-z0-9]+)(?:[/?&#]|$)/i.exec(
640
+ `${parsedUrl.pathname}${parsedUrl.search}`
641
+ );
642
+ if (!matched) {
643
+ return "";
644
+ }
645
+ return XHS_URL_FORMAT_EXTENSIONS.get(sanitizeFileExtension(matched[1])) || "";
646
+ }
647
+
648
+ function isExistingOutputPathError(error) {
649
+ return error?.code === "EEXIST" || error?.code === "EISDIR";
650
+ }
651
+
652
+ function shouldFallbackToCopyAfterLinkError(error) {
653
+ return ["EPERM", "ENOTSUP", "ENOSYS", "EOPNOTSUPP", "EXDEV"].includes(
654
+ error?.code
655
+ );
656
+ }
657
+
658
+ async function buildExistingOutputPathError(outputPath) {
659
+ const existingOutput = await stat(outputPath).catch(ignoreMissing);
660
+ if (existingOutput?.isDirectory()) {
661
+ return new Error("--output must be a file path for xhs download-media.");
662
+ }
663
+ return new Error("XHS media download output file already exists.");
664
+ }
665
+
666
+ function formatTimestamp(value) {
667
+ const date = value instanceof Date ? value : new Date(value);
668
+ const pad = (part) => String(part).padStart(2, "0");
669
+ return [
670
+ date.getFullYear(),
671
+ pad(date.getMonth() + 1),
672
+ pad(date.getDate()),
673
+ pad(date.getHours()),
674
+ pad(date.getMinutes()),
675
+ pad(date.getSeconds()),
676
+ ].join("");
677
+ }
678
+
679
+ function safeDecodeURIComponent(value) {
680
+ try {
681
+ return decodeURIComponent(value);
682
+ } catch {
683
+ return value;
684
+ }
685
+ }
686
+
687
+ function expandHome(path) {
688
+ if (!path || path === "~") {
689
+ return homedir();
690
+ }
691
+ if (path.startsWith("~/")) {
692
+ return join(homedir(), path.slice(2));
693
+ }
694
+ return path;
695
+ }
696
+
697
+ function ignoreMissing(error) {
698
+ if (error?.code === "ENOENT" || error?.code === "ENOTDIR") {
699
+ return undefined;
700
+ }
701
+ throw error;
702
+ }
703
+
704
+ function ignoreError() {}
705
+
706
+ function sleep(ms) {
707
+ if (!Number.isFinite(ms) || ms <= 0) {
708
+ return Promise.resolve();
709
+ }
710
+ return new Promise((resolvePromise) => {
711
+ setTimeout(resolvePromise, ms);
712
+ });
713
+ }
714
+
715
+ class XhsRetryableDownloadError extends Error {}
716
+
717
+ class XhsExpiredMediaLinkError extends Error {
718
+ constructor() {
719
+ super(
720
+ "XHS media link is unavailable or expired. Get a fresh media URL from the XHS detail result and retry."
721
+ );
722
+ }
723
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "socialdatax-skills",
3
- "version": "0.2.25",
3
+ "version": "0.2.27",
4
4
  "description": "SocialDataX / 社媒数据助手 skill installer and direct CLI helper for hosted platform social media MCP services.",
5
5
  "license": "MIT",
6
6
  "type": "module",