tt-help-cli-ycl 1.0.3 → 1.0.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.0.3",
+  "version": "1.0.4",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {
@@ -31,7 +31,7 @@
   },
   "repository": {
     "type": "git",
-    "url": "https://github.com/jsjhycl/tt-help-cli.git"
+    "url": "git+https://github.com/jsjhycl/tt-help-cli.git"
   },
   "bugs": {
     "url": "https://github.com/jsjhycl/tt-help-cli/issues"
@@ -41,4 +41,4 @@
     "playwright": "^1.59.1",
     "undici": "^8.1.0"
   }
-}
+}

package/src/lib/args.js

@@ -14,6 +14,7 @@ export function parseArgs() {
   let configAction = null;
   let configValue = null;
   let pipeMode = false;
+  let filterStr = null;
 
   for (let i = 0; i < args.length; i++) {
     const arg = args[i];
@@ -24,6 +25,8 @@ export function parseArgs() {
         : 100;
     } else if (arg === '--proxy') {
       customProxy = args[++i];
+    } else if (arg === '--filter') {
+      filterStr = args[++i];
     } else if (arg === 'config') {
       configAction = args[i + 1];
       if (configAction === 'set' || configAction === 'set-proxy' || configAction === 'set-browser') {
@@ -55,5 +58,5 @@ export function parseArgs() {
     urls.push(...lines);
   }
 
-  return { urls, outputFile, outputFormat, exploreCount, showConfig, showHelp, customProxy, configAction, configValue, pipeMode };
+  return { urls, outputFile, outputFormat, exploreCount, showConfig, showHelp, customProxy, configAction, configValue, pipeMode, filterStr };
 }

package/src/lib/constants.js

@@ -45,12 +45,18 @@ const HELP_TEXT = [
   '  --explore [count]      从 Explore 页面获取视频列表（默认: 100）',
   '  --pipe                 将 Explore 结果自动传给 URL 爬取',
   '  --proxy <地址>          临时指定代理地址',
+  '  --filter <条件>         过滤结果（格式: key=value&key2=value2）',
   '  -i, --input <file>     从文件读取 URL 列表（每行一个）',
   '  -o, --output <file>    指定输出文件（默认: tiktok_data.json）',
   '  -f, --format <fmt>     输出格式: json（默认）, table, raw',
   '  -c, --config           显示当前配置',
   '  -h, --help             显示帮助',
   '',
+  '过滤示例:',
+  '  --filter "ttSeller=true&verified=false"           过滤卖家且未认证',
+  '  --filter "locationCreated=DE,ES"                  过滤指定地区',
+  '  --filter "ttSeller=true&locationCreated=US"       组合条件',
+  '',
   '配置代理:',
   '  tt-help config set http://127.0.0.1:7890  设置代理',
   '  tt-help config show                        查看配置',
@@ -68,6 +74,7 @@ const HELP_TEXT = [
   '  tt-help -i urls.txt -o result.json       # 再爬取这些 URL',
   '  tt-help config set http://127.0.0.1:7890',
   '  tt-help https://www.tiktok.com/@username',
+  '  tt-help https://... --filter "ttSeller=true&locationCreated=DE"',
 ];
 
 const CONFIG_TEXT = [

package/src/lib/fetcher.js

@@ -2,16 +2,30 @@ import { fetch, ProxyAgent } from 'undici';
 import { DEFAULT_PROXY } from './constants.js';
 
 const HEADERS = {
-  'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
-  'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+  'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+  'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
+  'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
+  'Accept-Encoding': 'gzip, deflate, br',
+  'Connection': 'keep-alive',
+  'Upgrade-Insecure-Requests': '1',
+  'Sec-Fetch-Dest': 'document',
+  'Sec-Fetch-Mode': 'navigate',
+  'Sec-Fetch-Site': 'none',
+  'Sec-Fetch-User': '?1',
+  'Cache-Control': 'max-age=0',
 };
 
 export async function fetchHtml(url, proxyUrl) {
   const p = proxyUrl || DEFAULT_PROXY;
   const agent = new ProxyAgent(p);
   try {
-    const res = await fetch(url, { headers: HEADERS, dispatcher: agent });
-    return res.text();
+    const res = await fetch(url, { 
+      headers: HEADERS, 
+      dispatcher: agent,
+      redirect: 'follow',
+    });
+    const html = await res.text();
+    return html;
   } catch (err) {
     throw new Error(`请求 ${url} 失败，代理 ${p} 不可用`);
   }

package/src/lib/filter.js

@@ -0,0 +1,66 @@
+export function parseFilter(filterStr) {
+  if (!filterStr) return null;
+
+  const filter = {};
+  const pairs = filterStr.split('&');
+
+  for (const pair of pairs) {
+    const [key, value] = pair.split('=');
+    if (!key || value === undefined) continue;
+
+    const trimmedKey = key.trim();
+    const trimmedValue = value.trim();
+
+    // 处理布尔值
+    if (trimmedValue === 'true') {
+      filter[trimmedKey] = true;
+    } else if (trimmedValue === 'false') {
+      filter[trimmedKey] = false;
+    } else {
+      // 支持逗号分隔的多个值（如 locationCreated=DE,ES）
+      filter[trimmedKey] = trimmedValue.split(',').map(v => v.trim());
+    }
+  }
+
+  return Object.keys(filter).length > 0 ? filter : null;
+}
+
+export function applyFilter(results, filter) {
+  if (!filter || results.length === 0) return results;
+
+  return results.filter(item => {
+    for (const [key, expectedValue] of Object.entries(filter)) {
+      const actualValue = item[key];
+
+      // 如果字段不存在，过滤掉
+      if (actualValue === undefined || actualValue === null) {
+        return false;
+      }
+
+      // 数组值匹配（如 locationCreated=DE,ES）
+      if (Array.isArray(expectedValue)) {
+        if (!expectedValue.includes(String(actualValue))) {
+          return false;
+        }
+      }
+      // 布尔值或精确匹配
+      else if (actualValue !== expectedValue) {
+        return false;
+      }
+    }
+    return true;
+  });
+}
+
+export function formatFilterDescription(filter) {
+  if (!filter) return '';
+
+  const parts = Object.entries(filter).map(([key, value]) => {
+    if (Array.isArray(value)) {
+      return `${key}=${value.join(',')}`;
+    }
+    return `${key}=${value}`;
+  });
+
+  return parts.join(' & ');
+}

package/src/main.mjs

@@ -3,6 +3,7 @@ import { HELP_TEXT, CONFIG_TEXT, proxy, configFile, configPath, DEFAULT_PROXY, s
 import { fetchExplore } from './lib/explore.js';
 import { processUrl } from './lib/scrape.js';
 import { deduplicate, formatOutput } from './lib/output.js';
+import { parseFilter, applyFilter, formatFilterDescription } from './lib/filter.js';
 import { writeFileSync, readFileSync, existsSync } from 'fs';
 
 function showConfig(urls, outputFile) {
@@ -79,7 +80,7 @@ function cleanError(msg) {
     .trim();
 }
 
-async function runExplore(exploreCount, urls, proxyUrl, outputFile, outputFormat, isPipe) {
+async function runExplore(exploreCount, urls, proxyUrl, outputFile, outputFormat, isPipe, filter) {
   console.log(`\n代理: ${proxyUrl}`);
   console.log(`Explore 数量: ${exploreCount}`);
   if (urls.length > 0) {
@@ -97,7 +98,7 @@ async function runExplore(exploreCount, urls, proxyUrl, outputFile, outputFormat
       if (isPipe) {
         const videoUrls = exploreResults.map(r => r.url).filter(Boolean);
         if (videoUrls.length > 0) {
-          await runScrape(videoUrls, proxyUrl, outputFile, outputFormat);
+          await runScrape(videoUrls, proxyUrl, outputFile, outputFormat, filter);
           return;
         }
       }
@@ -142,8 +143,9 @@ async function runExplore(exploreCount, urls, proxyUrl, outputFile, outputFormat
   }
 
   const uniqueResults = deduplicate(allResults);
+  const filteredResults = applyFilter(uniqueResults, filter);
 
-  if (uniqueResults.length === 0) {
+  if (filteredResults.length === 0) {
     console.log('\n未获取到数据');
     if (outputFile) {
       writeFileSync(outputFile, '[]', 'utf-8');
@@ -151,7 +153,7 @@ async function runExplore(exploreCount, urls, proxyUrl, outputFile, outputFormat
     return;
   }
 
-  const output = formatOutput(uniqueResults, outputFormat);
+  const output = formatOutput(filteredResults, outputFormat);
 
   if (outputFile) {
     writeFileSync(outputFile, output, 'utf-8');
@@ -159,10 +161,15 @@ async function runExplore(exploreCount, urls, proxyUrl, outputFile, outputFormat
   } else {
     console.log(output);
   }
-  console.log(`\n共 ${uniqueResults.length} 个数据`);
+  
+  if (filter) {
+    console.log(`\n共 ${uniqueResults.length} 个数据，过滤后 ${filteredResults.length} 个（过滤条件: ${formatFilterDescription(filter)}）`);
+  } else {
+    console.log(`\n共 ${filteredResults.length} 个数据`);
+  }
 }
 
-async function runScrape(urls, proxyUrl, outputFile, outputFormat) {
+async function runScrape(urls, proxyUrl, outputFile, outputFormat, filter) {
   const allResults = [];
   const errors = [];
 
@@ -179,9 +186,10 @@ async function runScrape(urls, proxyUrl, outputFile, outputFormat) {
   console.log();
 
   const uniqueResults = deduplicate(allResults);
+  const filteredResults = applyFilter(uniqueResults, filter);
 
   if (errors.length > 0) {
-    if (uniqueResults.length === 0) {
+    if (filteredResults.length === 0) {
       const msg = errors[0].message;
       if (msg.includes('不可用') || msg.includes('连接被拒绝') || msg.includes('连接中断') ||
           msg.includes('超时') || msg.includes('无法解析')) {
@@ -218,7 +226,7 @@ async function runScrape(urls, proxyUrl, outputFile, outputFormat) {
     }
   }
 
-  const output = formatOutput(uniqueResults, outputFormat);
+  const output = formatOutput(filteredResults, outputFormat);
 
   if (outputFile) {
     writeFileSync(outputFile, output, 'utf-8');
@@ -226,12 +234,18 @@ async function runScrape(urls, proxyUrl, outputFile, outputFormat) {
   } else {
     console.log(output);
   }
-  console.log(`\n共 ${uniqueResults.length} 个用户的数据`);
+  
+  if (filter) {
+    console.log(`\n共 ${uniqueResults.length} 个数据，过滤后 ${filteredResults.length} 个（过滤条件: ${formatFilterDescription(filter)}）`);
+  } else {
+    console.log(`\n共 ${filteredResults.length} 个用户的数据`);
+  }
 }
 
 async function main() {
-  const { urls, outputFile, outputFormat, exploreCount, showConfig: showCfg, showHelp, customProxy, configAction, configValue, pipeMode } = parseArgs();
+  const { urls, outputFile, outputFormat, exploreCount, showConfig: showCfg, showHelp, customProxy, configAction, configValue, pipeMode, filterStr } = parseArgs();
   const proxyUrl = customProxy || proxy;
+  const filter = parseFilter(filterStr);
 
   if (showHelp) {
     showUsage();
@@ -254,9 +268,9 @@ async function main() {
   }
 
   if (exploreCount > 0) {
-    await runExplore(exploreCount, urls, proxyUrl, outputFile, outputFormat, pipeMode);
+    await runExplore(exploreCount, urls, proxyUrl, outputFile, outputFormat, pipeMode, filter);
   } else {
-    await runScrape(urls, proxyUrl, outputFile, outputFormat);
+    await runScrape(urls, proxyUrl, outputFile, outputFormat, filter);
   }
 }