tt-help-cli-ycl 1.0.5 → 1.0.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.0.5",
+  "version": "1.0.6",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {

package/src/lib/io.js

@@ -1,5 +1,7 @@
 import { writeFileSync, readFileSync } from 'fs';
 
+let lastBarCount = 0;
+
 export function writeOutput(data, outputFile) {
   const output = JSON.stringify(data, null, 2);
   const target = outputFile || 'tiktok_data.json';
@@ -11,3 +13,64 @@ export function readUrlFile(filePath) {
   const content = readFileSync(filePath, 'utf-8');
   return content.split(/\r?\n/).map(l => l.trim()).filter(l => l.startsWith('http'));
 }
+
+export function createProgressBar(current, total, maxWidth = 30) {
+  const filled = Math.round((current / total) * maxWidth);
+  return '█'.repeat(filled).padEnd(maxWidth);
+}
+
+export function calculateConcurrency(total) {
+  return Math.min(5, Math.max(1, Math.floor(total / 10)), total);
+}
+
+export function extractUrlDisplay(url) {
+  try {
+    const pathname = new URL(url).pathname;
+    const parts = pathname.split('/').filter(Boolean);
+    return parts.slice(-2).join('/');
+  } catch {
+    return url;
+  }
+}
+
+export function createMultiProgressBars(count) {
+  return Array.from({ length: count }, () => ({
+    current: 0,
+    total: 0,
+    status: 'pending',
+    url: '',
+  }));
+}
+
+export function renderMultiProgressBars(bars, maxWidth = 30) {
+  const activeBars = bars.filter(bar => bar.total > 0);
+  
+  if (activeBars.length === 0) return;
+  
+  const lines = activeBars.map((bar) => {
+    const prog = createProgressBar(bar.current, bar.total, maxWidth);
+    const icon = bar.status === 'done' ? '✓' :
+                 bar.status === 'error' ? '' : '⟳';
+    const urlDisplay = bar.url ? extractUrlDisplay(bar.url) : '';
+    return `  [${prog}] ${bar.current}/${bar.total} ${icon} ${urlDisplay}`;
+  });
+  
+  const output = lines.join('\n');
+  
+  if (lastBarCount > 0) {
+    process.stdout.write(`\x1b[${lastBarCount}A`);
+  }
+  
+  process.stdout.write('\x1b[0J');
+  process.stdout.write(output + '\n');
+  
+  lastBarCount = activeBars.length;
+}
+
+export function clearProgressBars() {
+  if (lastBarCount > 0) {
+    process.stdout.write(`\x1b[${lastBarCount}A`);
+    process.stdout.write('\x1b[0J');
+    lastBarCount = 0;
+  }
+}

package/src/main.mjs

@@ -4,6 +4,7 @@ import { fetchExplore } from './lib/explore.js';
 import { processUrl } from './lib/scrape.js';
 import { deduplicate, formatOutput } from './lib/output.js';
 import { parseFilter, applyFilter, formatFilterDescription } from './lib/filter.js';
+import { createProgressBar, calculateConcurrency, createMultiProgressBars, renderMultiProgressBars, clearProgressBars } from './lib/io.js';
 import { writeFileSync, readFileSync, existsSync } from 'fs';
 
 function showConfig(urls, outputFile) {
@@ -72,6 +73,10 @@ function handleConfig(action, value) {
   process.exit(1);
 }
 
+function randomDelay() {
+  return new Promise(r => setTimeout(r, Math.random() * 600 + 200));
+}
+
 function cleanError(msg) {
   return msg
     .replace(/\x1b\[[0-9;]*m/g, '')
@@ -112,16 +117,46 @@ async function runExplore(exploreCount, urls, proxyUrl, outputFile, outputFormat
   if (urls.length > 0) {
     const errors = [];
 
-    for (let i = 0; i < urls.length; i++) {
-      const bar = '█'.repeat(i + 1).padEnd(urls.length);
-      process.stdout.write(`\r  [${bar}] ${i + 1}/${urls.length}`);
-      try {
-        const results = await processUrl(urls[i], proxyUrl);
-        allResults.push(...results);
-      } catch (err) {
-        errors.push({ url: urls[i], message: err.message });
+    const concurrency = calculateConcurrency(urls.length);
+    const bars = createMultiProgressBars(concurrency);
+
+    const slots = Array.from({ length: concurrency }, () => []);
+    urls.forEach((url, i) => slots[i % concurrency].push(url));
+
+    bars.forEach((bar, i) => {
+      bar.total = slots[i].length;
+      bar.status = slots[i].length > 0 ? 'running' : 'done';
+    });
+
+    renderMultiProgressBars(bars);
+
+    const workers = slots.map(async (slotUrls, slotIndex) => {
+      for (const url of slotUrls) {
+        bars[slotIndex].url = url;
+        renderMultiProgressBars(bars);
+
+        await randomDelay();
+
+        try {
+          const results = await processUrl(url, proxyUrl);
+          allResults.push(...results);
+          bars[slotIndex].current++;
+          bars[slotIndex].status = 'running';
+        } catch (err) {
+          errors.push({ url, message: err.message });
+          bars[slotIndex].current++;
+          bars[slotIndex].status = 'error';
+        }
+
+        renderMultiProgressBars(bars);
       }
-    }
+      bars[slotIndex].status = bars[slotIndex].current === bars[slotIndex].total ? 'done' : 'error';
+      renderMultiProgressBars(bars);
+    });
+
+    await Promise.all(workers);
+
+    clearProgressBars();
     console.log();
 
     if (errors.length > 0) {
@@ -173,16 +208,52 @@ async function runScrape(urls, proxyUrl, outputFile, outputFormat, filter) {
   const allResults = [];
   const errors = [];
 
-  for (let i = 0; i < urls.length; i++) {
-    const bar = '█'.repeat(i + 1).padEnd(urls.length);
-    process.stdout.write(`\r  [${bar}] ${i + 1}/${urls.length}`);
-    try {
-      const results = await processUrl(urls[i], proxyUrl);
-      allResults.push(...results);
-    } catch (err) {
-      errors.push({ url: urls[i], message: err.message });
+  if (urls.length === 0) {
+    console.log('\n未获取到数据');
+    if (outputFile) {
+      writeFileSync(outputFile, '[]', 'utf-8');
     }
+    return;
   }
+
+  const concurrency = calculateConcurrency(urls.length);
+  const bars = createMultiProgressBars(concurrency);
+
+  const slots = Array.from({ length: concurrency }, () => []);
+  urls.forEach((url, i) => slots[i % concurrency].push(url));
+
+  bars.forEach((bar, i) => {
+    bar.total = slots[i].length;
+    bar.status = slots[i].length > 0 ? 'running' : 'done';
+  });
+
+  renderMultiProgressBars(bars);
+
+  const workers = slots.map(async (slotUrls, slotIndex) => {
+    for (const url of slotUrls) {
+      bars[slotIndex].url = url;
+      renderMultiProgressBars(bars);
+
+      try {
+        const results = await processUrl(url, proxyUrl);
+        allResults.push(...results);
+        bars[slotIndex].current++;
+        bars[slotIndex].status = 'running';
+      } catch (err) {
+        errors.push({ url, message: err.message });
+        bars[slotIndex].current++;
+        bars[slotIndex].status = 'error';
+      }
+
+      renderMultiProgressBars(bars);
+    }
+    bars[slotIndex].status = bars[slotIndex].current === bars[slotIndex].total ? 'done' : 'error';
+    renderMultiProgressBars(bars);
+  });
+
+  await Promise.all(workers);
+
+  clearProgressBars();
   console.log();
 
   const uniqueResults = deduplicate(allResults);