socialdatax-skills 0.2.6 → 0.2.7

package/README.md

@@ -48,7 +48,7 @@ Common search phrases for this skill package:
 - Kuaishou current platform MCP registry name: `com.52choujiang/kuaishou-insights`
 - Kuaishou future platform MCP registry name: `com.socialdatax/kuaishou-insights`
 - Unified MCP registry name: none; this package installs skills and calls explicit platform services.
-- Current public capability version: `0.2.6`
+- Current public capability version: `0.2.7`
 
 ## Direct CLI
 
@@ -247,7 +247,7 @@ Current Douyin workflows include:
 - Search related Douyin works by keyword.
 - Resolve a Douyin content page link, short link, share text, or aweme_id into structured work details.
 - Fetch paginated first-level comments for comment analysis.
-- Fetch paginated replies under a first-level comment.
+- Fetch paginated replies under a first-level comment; pass both `aweme_id` and `comment_id`, and use `page_token` to continue pagination.
 - Continue Douyin comment and reply pagination only when `next_page_token` is non-empty; an empty string means there are no more comments or replies to request.
 - Read creator profile data from a profile link, short link, share text, or sec_user_id.
 - Fetch paginated creator work lists from a sec_user_id, profile link, short link, or share text.
@@ -279,7 +279,7 @@ Current Douyin workflows include:
 | `douyin_get_video_detail_by_url` | Resolve a Douyin content page link, short link, or share text into structured Douyin work details. |
 | `douyin_get_video_comments_by_aweme_id` | Fetch paginated first-level comments when the caller already has an aweme_id. |
 | `douyin_get_video_comments_by_url` | Fetch paginated first-level comments directly from a Douyin content page URL, short link, or share text. |
-| `douyin_get_video_comment_replies_by_comment_id` | Fetch paginated replies under a first-level Douyin comment by aweme_id and comment_id. |
+| `douyin_get_video_comment_replies_by_comment_id` | Fetch paginated replies under a first-level Douyin comment; pass both aweme_id and comment_id, and use page_token to continue pagination. |
 | `douyin_get_user_info_by_sec_user_id` | Fetch creator profile data when the caller already has a sec_user_id. |
 | `douyin_get_user_info_by_profile_url` | Resolve a profile link, short link, or share text into creator profile data. |
 | `douyin_get_user_posted_videos_by_sec_user_id` | Fetch a paginated list of works published by a creator when the caller already has a sec_user_id. |

package/cli.mjs

@@ -7,7 +7,7 @@ import { dirname, join, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 
 const PACKAGE_NAME = "socialdatax-skills";
-const PACKAGE_VERSION = "0.2.6";
+const PACKAGE_VERSION = "0.2.7";
 const PACKAGE_SPEC = `${PACKAGE_NAME}@latest`;
 const LOG_PREFIX = `[${PACKAGE_NAME}]`;
 const MIN_NODE_VERSION = "20.18.1";
@@ -339,7 +339,7 @@ const PLATFORMS = {
       },
       {
         name: "douyin_get_video_comment_replies_by_comment_id",
-        description: "Fetch paginated replies under a first-level Douyin comment by aweme_id and comment_id.",
+        description: "Fetch paginated replies under a first-level Douyin comment; pass both aweme_id and comment_id, and use page_token to continue pagination.",
       },
       {
         name: "douyin_get_user_info_by_sec_user_id",
@@ -1134,7 +1134,7 @@ function printHelp() {
   console.log(
     `  npx -y ${PACKAGE_SPEC} douyin replies --aweme-id "<aweme_id>" --comment-id "<comment_id>" --pretty`
   );
-  console.log("      Call the Douyin comment replies tool directly and print JSON.");
+  console.log("      Call the Douyin comment replies tool with aweme_id and comment_id; use page_token for pagination.");
   console.log("");
   console.log(`  npx -y ${PACKAGE_SPEC} douyin user-info --sec-user-id "<sec_user_id>" --pretty`);
   console.log("      Call the Douyin creator profile tool directly and print JSON.");
@@ -1945,6 +1945,7 @@ async function collectPaginatedDirectData(
 ) {
   const pageLimit = pagination.all ? Number.POSITIVE_INFINITY : pagination.pages || 1;
   const collectedItems = [];
+  const itemDedupeState = createPaginatedItemDedupeState();
   const seenNextMarkers = initialPaginationMarkers(operation);
   let pageCount = 0;
   let lastPageData;
@@ -1958,7 +1959,11 @@ async function collectPaginatedDirectData(
     lastPageData = pageData;
     parentContextData = mergeParentContextData(parentContextData, pageData);
 
-    const pageItems = directPageItems(pageData);
+    const pageItems = uniquePaginatedPageItems(
+      operation,
+      directPageItems(pageData),
+      itemDedupeState
+    );
     const candidateItems = itemsForRemainingLimit(
       pageItems,
       collectedItems.length,
@@ -2018,6 +2023,135 @@ function directPageItems(pageData) {
   return Array.isArray(pageData?.items) ? pageData.items : [];
 }
 
+function createPaginatedItemDedupeState() {
+  return { seenKeys: new Set() };
+}
+
+function uniquePaginatedPageItems(operation, items, itemDedupeState) {
+  const uniqueItems = [];
+  for (const item of items) {
+    const dedupeKeys = paginatedItemDedupeKeys(operation, item);
+    if (dedupeKeys.length === 0) {
+      uniqueItems.push(item);
+      continue;
+    }
+    if (dedupeKeys.some((key) => itemDedupeState.seenKeys.has(key))) {
+      recordPaginatedItemDedupeKeys(itemDedupeState, dedupeKeys);
+      continue;
+    }
+    recordPaginatedItemDedupeKeys(itemDedupeState, dedupeKeys);
+    uniqueItems.push(item);
+  }
+  return uniqueItems;
+}
+
+function recordPaginatedItemDedupeKeys(itemDedupeState, dedupeKeys) {
+  for (const key of dedupeKeys) {
+    itemDedupeState.seenKeys.add(key);
+  }
+}
+
+function paginatedItemDedupeKeys(operation, item) {
+  if (shouldDeduplicateCommentItems(operation)) {
+    const commentId = itemStringField(item, "comment_id");
+    return commentId ? [`comment:${commentId}`] : [];
+  }
+  if (!shouldDeduplicateContentItems(operation)) {
+    return [];
+  }
+  const keys = [];
+  const itemId = contentItemId(operation, item);
+  if (itemId) {
+    keys.push(`${operation.operation}:${operation.platform.id}:id:${itemId}`);
+  }
+  const xhsFingerprint = xhsContentItemFingerprint(operation, item);
+  if (xhsFingerprint) {
+    keys.push(`${operation.operation}:xhs:fingerprint:${xhsFingerprint}`);
+  }
+  return keys;
+}
+
+function shouldDeduplicateCommentItems(operation) {
+  return ["comments", "replies", "sub-comments"].includes(operation.operation);
+}
+
+function shouldDeduplicateContentItems(operation) {
+  return ["search", "user-posts", "user-series"].includes(operation.operation);
+}
+
+function contentItemId(operation, item) {
+  switch (operation.platform.id) {
+    case "xhs":
+      return itemStringField(item, "note_id");
+    case "douyin":
+      if (operation.operation === "user-series") {
+        return itemStringField(item, "series_id");
+      }
+      return itemStringField(item, "aweme_id");
+    case "kuaishou":
+      return itemStringField(item, "photo_id");
+    default:
+      return undefined;
+  }
+}
+
+function xhsContentItemFingerprint(operation, item) {
+  if (
+    operation.platform.id !== "xhs" ||
+    !shouldDeduplicateContentItems(operation)
+  ) {
+    return undefined;
+  }
+  const authorId = itemStringField(item?.author, "user_id");
+  const publishTime = itemStringField(item, "publish_time");
+  const publishTimeNumber = Number(publishTime);
+  const coverToken = xhsImageUrlToken(item?.cover_image_url);
+  if (
+    !authorId ||
+    !publishTime ||
+    !Number.isFinite(publishTimeNumber) ||
+    publishTimeNumber <= 0 ||
+    !coverToken
+  ) {
+    return undefined;
+  }
+  return `${authorId}\t${publishTime}\t${coverToken}`;
+}
+
+function xhsImageUrlToken(imageUrl) {
+  const urlValue = stringValue(imageUrl);
+  if (!urlValue) {
+    return undefined;
+  }
+  let pathname;
+  try {
+    pathname = new URL(urlValue).pathname;
+  } catch {
+    return undefined;
+  }
+  let token = pathname.replace(/^\/+/, "");
+  const webpicContentIndex = token.indexOf("/c/");
+  if (webpicContentIndex !== -1) {
+    token = token.slice(webpicContentIndex + 3);
+  }
+  token = token.split("!", 1)[0];
+  return token || undefined;
+}
+
+function itemStringField(item, field) {
+  if (!item || typeof item !== "object") {
+    return undefined;
+  }
+  return stringValue(item[field]);
+}
+
+function stringValue(value) {
+  if (value === undefined || value === null || value === "") {
+    return undefined;
+  }
+  return String(value);
+}
+
 function mergeParentContextData(current, pageData) {
   return {
     ...current,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "socialdatax-skills",
-  "version": "0.2.6",
+  "version": "0.2.7",
   "description": "SocialDataX / 社媒数据助手 skill installer and direct CLI helper for hosted platform social media MCP services.",
   "license": "MIT",
   "type": "module",

package/skills/media-comments/SKILL.md

@@ -96,7 +96,7 @@ If MCP tools are already available in the current agent, use one of these tools:
 - `xhs_get_note_sub_comments_by_comment_id`: use when the complete 24-character lowercase hexadecimal `note_id` and first-level comment ID are known; do not pass only a note ID prefix.
 - `douyin_get_video_comments_by_aweme_id`: use when the aweme_id is known.
 - `douyin_get_video_comments_by_url`: use for Douyin content page URLs, short links, or share text; do not pass playback URLs such as `video.play_url`.
-- `douyin_get_video_comment_replies_by_comment_id`: use when the aweme_id and first-level comment ID are known.
+- `douyin_get_video_comment_replies_by_comment_id`: use when both aweme_id and first-level comment ID are known; use page_token to continue pagination.
 
 Comment pagination uses opaque `page_token` values. Pass the complete returned `next_page_token` back unchanged for the same content item or comment chain. Do not modify, truncate, redact, mask, omit, normalize, rebuild, generate, or replace the middle with ellipses. Prefer CLI `--pages`, `--all`, and `--include-replies` when the user asks for multiple pages or a full first-level plus second-level comment tree.
 For Douyin comments and replies, continue only when `next_page_token` is non-empty; an empty string means there are no more comments or replies to request.