stealth-cli 0.5.0

package/src/extractors/github.js

@@ -0,0 +1,103 @@
+/**
+ * GitHub Search extractor
+ */
+
+export const name = 'github';
+
+export function canHandle(url) {
+  return /github\.com\/search/.test(url);
+}
+
+export async function waitForResults(page) {
+  const selectors = [
+    '[data-testid="results-list"]',
+    '.repo-list',
+    '[data-testid="result"]',
+    '.search-title',
+    '.Box-row',                    // Modern GitHub search layout
+    'div[data-testid="result"]',
+  ];
+
+  for (const sel of selectors) {
+    try {
+      await page.waitForSelector(sel, { timeout: 3000 });
+      return;
+    } catch {}
+  }
+
+  // GitHub search can be slow, give it more time
+  await new Promise((r) => setTimeout(r, 3000));
+}
+
+export async function extractResults(page, maxResults = 10) {
+  await waitForResults(page);
+
+  return page.evaluate((max) => {
+    const results = [];
+
+    // Modern GitHub search (React)
+    const items = document.querySelectorAll(
+      '[data-testid="results-list"] > div, .search-title, .repo-list-item, [data-testid="result"]',
+    );
+
+    for (const item of items) {
+      if (results.length >= max) break;
+
+      // Repository link — try multiple selectors
+      const linkEl = item.querySelector(
+        'a[href*="github.com/"]:has(.search-match), a.v-align-middle, a[data-testid="result-title-link"]',
+      );
+
+      if (linkEl) {
+        const descEl = item.querySelector('p, .mb-1, [data-testid="result-description"]');
+        const langEl = item.querySelector('[itemprop="programmingLanguage"], span.repo-language-color + span');
+        const starsEl = item.querySelector('a[href*="/stargazers"], [aria-label*="star"]');
+        const updatedEl = item.querySelector('relative-time, [datetime]');
+
+        results.push({
+          title: linkEl.textContent?.trim() || '',
+          url: linkEl.href,
+          description: descEl?.textContent?.trim() || '',
+          language: langEl?.textContent?.trim() || '',
+          stars: starsEl?.textContent?.trim() || '',
+          updated: updatedEl?.getAttribute('datetime') || updatedEl?.textContent?.trim() || '',
+        });
+        continue;
+      }
+
+      // Fallback: any link to a repo pattern /user/repo
+      const anyLink = item.querySelector('a[href*="github.com/"]');
+      if (anyLink && anyLink.href.match(/github\.com\/[\w-]+\/[\w.-]+$/)) {
+        const descEl = item.querySelector('p, .mb-1');
+        results.push({
+          title: anyLink.textContent?.trim() || '',
+          url: anyLink.href,
+          description: descEl?.textContent?.trim() || '',
+        });
+      }
+    }
+
+    // Strategy 2: Fallback — grab all repo-like links on the page
+    if (results.length === 0) {
+      const allLinks = document.querySelectorAll('a[href]');
+      const seenUrls = new Set();
+      for (const link of allLinks) {
+        if (results.length >= max) break;
+        const href = link.href;
+        if (
+          href?.match(/github\.com\/[\w-]+\/[\w.-]+$/) &&
+          !href.includes('/search') &&
+          !seenUrls.has(href)
+        ) {
+          seenUrls.add(href);
+          const text = link.textContent?.trim();
+          if (text && text.length > 2 && text.includes('/')) {
+            results.push({ title: text, url: href, description: '' });
+          }
+        }
+      }
+    }
+
+    return results;
+  }, maxResults);
+}

package/src/extractors/google.js

@@ -0,0 +1,173 @@
+/**
+ * Google Search extractor
+ *
+ * Strategy:
+ *   1. Navigate to google.com homepage first (not direct search URL)
+ *   2. Type query into search box like a human
+ *   3. Wait for async SERP rendering
+ *   4. Extract structured results from multiple layout variants
+ */
+
+import { humanType, humanScroll, randomDelay } from '../humanize.js';
+
+export const name = 'google';
+
+export function canHandle(url) {
+  return /google\.\w+\/search/.test(url);
+}
+
+/**
+ * Perform Google search with human-like behavior
+ * Instead of direct URL, simulates typing into search box
+ *
+ * @param {Page} page - Playwright page
+ * @param {string} query - Search query
+ * @returns {boolean} success
+ */
+export async function humanSearch(page, query) {
+  // Step 1: Visit Google homepage
+  await page.goto('https://www.google.com', {
+    waitUntil: 'domcontentloaded',
+    timeout: 15000,
+  });
+  await randomDelay(800, 2000);
+
+  // Step 2: Handle cookie consent (EU/UK)
+  try {
+    const consentBtn = page.locator(
+      'button:has-text("Accept all"), button:has-text("Accept"), button:has-text("Agree"), button:has-text("I agree"), [id*="accept"], [aria-label*="Accept"]',
+    );
+    if (await consentBtn.first().isVisible({ timeout: 1500 })) {
+      await consentBtn.first().click({ timeout: 2000 });
+      await randomDelay(500, 1000);
+    }
+  } catch {}
+
+  // Step 3: Type query with human timing
+  try {
+    await humanType(page, 'textarea[name="q"], input[name="q"]', query, {
+      pressEnter: true,
+    });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Wait for Google results to render (they load asynchronously)
+ */
+export async function waitForResults(page) {
+  // Multiple selectors for different Google layouts
+  const selectors = [
+    '#rso h3',
+    '#search h3',
+    '#rso [data-snhf]',
+    '#rso a[href]:not([href^="/search"])',
+    '.g h3',
+  ];
+
+  for (const sel of selectors) {
+    try {
+      await page.waitForSelector(sel, { timeout: 3000 });
+      return;
+    } catch {}
+  }
+
+  // Last resort: wait for any meaningful content
+  await randomDelay(2000, 3000);
+}
+
+/**
+ * Check if Google blocked us (CAPTCHA / sorry page)
+ */
+export function isBlocked(url) {
+  return url.includes('/sorry/') || url.includes('consent.google') || url.includes('/recaptcha/');
+}
+
+/**
+ * Extract Google search results
+ */
+export async function extractResults(page, maxResults = 10) {
+  await waitForResults(page);
+
+  return page.evaluate((max) => {
+    const results = [];
+
+    // Strategy 1: Standard search result cards
+    const items = document.querySelectorAll(
+      '#rso .g, #rso [data-sokoban-container], #rso [data-snhf], #search .g, .MjjYud',
+    );
+
+    for (const item of items) {
+      if (results.length >= max) break;
+
+      const linkEl = item.querySelector(
+        'a[href]:not([href^="/search"]):not([href^="#"]):not([href*="google.com/search"])',
+      );
+      const titleEl = item.querySelector('h3');
+      const snippetEl = item.querySelector(
+        '[data-sncf], .VwiC3b, [style*="-webkit-line-clamp"], .IsZvec, .lEBKkf',
+      );
+
+      if (linkEl && titleEl) {
+        const href = linkEl.href;
+        if (href && href.startsWith('http') && !href.includes('google.com/search')) {
+          results.push({
+            title: titleEl.textContent?.trim() || '',
+            url: href,
+            snippet: snippetEl?.textContent?.trim() || '',
+          });
+        }
+      }
+    }
+
+    // Strategy 2: Fallback — look for any external links with headings
+    if (results.length === 0) {
+      const fallbackLinks = document.querySelectorAll('#rso a[href], #search a[href]');
+      const seenUrls = new Set();
+
+      for (const link of fallbackLinks) {
+        if (results.length >= max) break;
+        const href = link.href;
+        if (
+          href?.startsWith('http') &&
+          !href.includes('google.') &&
+          !href.includes('gstatic.') &&
+          !seenUrls.has(href)
+        ) {
+          seenUrls.add(href);
+          const heading = link.querySelector('h3, h2, [role="heading"]');
+          const text = heading?.textContent?.trim() || link.textContent?.trim();
+          if (text && text.length > 3) {
+            results.push({ title: text.slice(0, 200), url: href, snippet: '' });
+          }
+        }
+      }
+    }
+
+    return results;
+  }, maxResults);
+}
+
+/**
+ * Extract "People also ask" questions
+ */
+export async function extractPeopleAlsoAsk(page, maxResults = 5) {
+  return page.evaluate((max) => {
+    const questions = [];
+    const items = document.querySelectorAll(
+      '[data-sgrd] [role="button"], .related-question-pair, [jsname="Cpkphb"]',
+    );
+
+    for (const item of items) {
+      if (questions.length >= max) break;
+      const text = item.textContent?.trim();
+      if (text && text.length > 10) {
+        questions.push(text);
+      }
+    }
+
+    return questions;
+  }, maxResults);
+}

package/src/extractors/index.js

@@ -0,0 +1,55 @@
+/**
+ * Extractor registry — auto-selects the best extractor for a URL/engine
+ */
+
+import * as google from './google.js';
+import * as duckduckgo from './duckduckgo.js';
+import * as bing from './bing.js';
+import * as github from './github.js';
+import * as youtube from './youtube.js';
+import * as base from './base.js';
+
+const extractors = [google, duckduckgo, bing, github, youtube];
+
+// Map engine names to extractors
+const engineMap = {
+  google: google,
+  duckduckgo: duckduckgo,
+  bing: bing,
+  github: github,
+  youtube: youtube,
+};
+
+/**
+ * Get the best extractor for a given engine name
+ *
+ * @param {string} engine - Engine name (google, bing, etc.)
+ * @returns {object} Extractor module
+ */
+export function getExtractorByEngine(engine) {
+  return engineMap[engine.toLowerCase()] || base;
+}
+
+/**
+ * Get the best extractor for a given URL
+ *
+ * @param {string} url - Page URL
+ * @returns {object} Extractor module
+ */
+export function getExtractorByUrl(url) {
+  for (const extractor of extractors) {
+    if (extractor.canHandle(url)) {
+      return extractor;
+    }
+  }
+  return base;
+}
+
+/**
+ * List all available extractors
+ */
+export function listExtractors() {
+  return [...extractors.map((e) => e.name), base.name];
+}
+
+export { google, duckduckgo, bing, github, youtube, base };

package/src/extractors/youtube.js

@@ -0,0 +1,87 @@
+/**
+ * YouTube Search extractor
+ */
+
+export const name = 'youtube';
+
+export function canHandle(url) {
+  return /youtube\.com\/results/.test(url);
+}
+
+export async function waitForResults(page) {
+  try {
+    await page.waitForSelector(
+      'ytd-video-renderer, ytd-channel-renderer, #contents ytd-item-section-renderer',
+      { timeout: 6000 },
+    );
+  } catch {}
+
+  // YouTube loads results dynamically; give extra time
+  await new Promise((r) => setTimeout(r, 1500));
+}
+
+export async function extractResults(page, maxResults = 10) {
+  await waitForResults(page);
+
+  return page.evaluate((max) => {
+    const results = [];
+
+    // Video renderers
+    const items = document.querySelectorAll(
+      'ytd-video-renderer, ytd-channel-renderer, ytd-playlist-renderer',
+    );
+
+    for (const item of items) {
+      if (results.length >= max) break;
+
+      const isChannel = item.tagName === 'YTD-CHANNEL-RENDERER';
+      const isPlaylist = item.tagName === 'YTD-PLAYLIST-RENDERER';
+
+      // Video title link
+      const titleLink = item.querySelector(
+        'a#video-title, h3 a, a.channel-link, a[href*="/watch"], a[href*="/playlist"]',
+      );
+      if (!titleLink) continue;
+
+      const title = titleLink.textContent?.trim() || titleLink.getAttribute('title') || '';
+      const url = titleLink.href;
+
+      if (!url || !title) continue;
+
+      // Channel name
+      const channelEl = item.querySelector(
+        'ytd-channel-name a, .ytd-channel-name a, #channel-info a, #text.ytd-channel-name',
+      );
+
+      // View count and upload date
+      const metaItems = item.querySelectorAll(
+        '#metadata-line span, .inline-metadata-item, .ytd-video-meta-block span',
+      );
+      const views = metaItems[0]?.textContent?.trim() || '';
+      const uploadDate = metaItems[1]?.textContent?.trim() || '';
+
+      // Duration
+      const durationEl = item.querySelector(
+        'span.ytd-thumbnail-overlay-time-status-renderer, badge-shape .badge-shape-wiz__text',
+      );
+
+      // Thumbnail
+      const thumbEl = item.querySelector('img#img, ytd-thumbnail img');
+
+      const result = {
+        type: isChannel ? 'channel' : isPlaylist ? 'playlist' : 'video',
+        title,
+        url,
+        channel: channelEl?.textContent?.trim() || '',
+        views,
+        uploadDate,
+        duration: durationEl?.textContent?.trim() || '',
+        thumbnail: thumbEl?.src || '',
+      };
+
+      results.push(result);
+    }
+
+    return results;
+  }, maxResults);
+}

package/src/humanize.js

@@ -0,0 +1,210 @@
+/**
+ * Human behavior simulation — makes browser automation less detectable
+ */
+
+/**
+ * Random delay between min and max ms
+ */
+export function randomDelay(min = 200, max = 800) {
+  const delay = min + Math.random() * (max - min);
+  return new Promise((resolve) => setTimeout(resolve, delay));
+}
+
+/**
+ * Simulate human-like scrolling pattern
+ * - Variable speed
+ * - Occasional pauses
+ * - Sometimes scroll back up slightly
+ */
+export async function humanScroll(page, opts = {}) {
+  const { scrolls = 3, direction = 'down' } = opts;
+
+  for (let i = 0; i < scrolls; i++) {
+    // Variable scroll amount (200-600px)
+    const amount = 200 + Math.random() * 400;
+    const delta = direction === 'up' ? -amount : amount;
+
+    await page.mouse.wheel(0, delta);
+
+    // Random pause between scrolls (300-1200ms)
+    await randomDelay(300, 1200);
+
+    // 20% chance to scroll back slightly (human-like hesitation)
+    if (Math.random() < 0.2 && i < scrolls - 1) {
+      const backAmount = 50 + Math.random() * 100;
+      await page.mouse.wheel(0, -backAmount * Math.sign(delta));
+      await randomDelay(200, 500);
+    }
+  }
+}
+
+/**
+ * Move mouse along a bezier curve (more natural than straight line)
+ */
+export async function humanMouseMove(page, targetX, targetY, opts = {}) {
+  const { steps = 15 } = opts;
+
+  // Get current or random starting position
+  const startX = 100 + Math.random() * 400;
+  const startY = 100 + Math.random() * 300;
+
+  // Generate control points for bezier curve
+  const cp1x = startX + (targetX - startX) * 0.3 + (Math.random() - 0.5) * 100;
+  const cp1y = startY + (targetY - startY) * 0.1 + (Math.random() - 0.5) * 100;
+  const cp2x = startX + (targetX - startX) * 0.7 + (Math.random() - 0.5) * 80;
+  const cp2y = startY + (targetY - startY) * 0.9 + (Math.random() - 0.5) * 80;
+
+  for (let i = 0; i <= steps; i++) {
+    const t = i / steps;
+
+    // Cubic bezier interpolation
+    const x = cubicBezier(t, startX, cp1x, cp2x, targetX);
+    const y = cubicBezier(t, startY, cp1y, cp2y, targetY);
+
+    await page.mouse.move(x, y);
+
+    // Variable speed: slower at start and end (ease-in-out)
+    const speedFactor = Math.sin(t * Math.PI); // 0→1→0
+    const delay = 5 + (1 - speedFactor) * 15;
+    await new Promise((r) => setTimeout(r, delay));
+  }
+}
+
+/**
+ * Type text with human-like timing
+ * - Variable delay between keystrokes
+ * - Occasional longer pauses (thinking)
+ * - Optionally make typos and correct them
+ */
+export async function humanType(page, selector, text, opts = {}) {
+  const { typoRate = 0, pressEnter = false } = opts;
+
+  // Click the input first
+  await page.click(selector, { timeout: 5000 });
+  await randomDelay(200, 500);
+
+  // Clear existing content
+  await page.fill(selector, '');
+  await randomDelay(100, 300);
+
+  // Type character by character
+  for (let i = 0; i < text.length; i++) {
+    const char = text[i];
+
+    // Simulate typo (if enabled)
+    if (typoRate > 0 && Math.random() < typoRate) {
+      // Type wrong character
+      const wrongChar = String.fromCharCode(char.charCodeAt(0) + (Math.random() > 0.5 ? 1 : -1));
+      await page.keyboard.type(wrongChar, { delay: 30 + Math.random() * 50 });
+      await randomDelay(100, 300);
+
+      // Delete it
+      await page.keyboard.press('Backspace');
+      await randomDelay(50, 150);
+    }
+
+    // Type correct character
+    await page.keyboard.type(char, { delay: 30 + Math.random() * 80 });
+
+    // Occasional longer pause (thinking / reading)
+    if (Math.random() < 0.05) {
+      await randomDelay(300, 800);
+    }
+  }
+
+  if (pressEnter) {
+    await randomDelay(300, 600);
+    await page.keyboard.press('Enter');
+  }
+}
+
+/**
+ * Simulate human click with mouse movement
+ */
+export async function humanClick(page, selector, opts = {}) {
+  const { timeout = 5000 } = opts;
+
+  const element = page.locator(selector);
+  const box = await element.boundingBox({ timeout });
+
+  if (!box) {
+    // Fallback to regular click
+    await element.click({ timeout });
+    return;
+  }
+
+  // Move to element with slight offset (humans don't click exact center)
+  const offsetX = (Math.random() - 0.5) * box.width * 0.4;
+  const offsetY = (Math.random() - 0.5) * box.height * 0.3;
+  const targetX = box.x + box.width / 2 + offsetX;
+  const targetY = box.y + box.height / 2 + offsetY;
+
+  await humanMouseMove(page, targetX, targetY);
+  await randomDelay(50, 150);
+
+  // Mouse down → small pause → mouse up (human click duration)
+  await page.mouse.down();
+  await randomDelay(30, 80);
+  await page.mouse.up();
+}
+
+/**
+ * Warmup routine - perform natural browsing actions before target URL
+ * This helps establish a "normal" browsing fingerprint
+ */
+export async function warmup(page, opts = {}) {
+  const { sites = null, scrollAfter = true } = opts;
+
+  const defaultSites = [
+    'https://www.wikipedia.org',
+    'https://www.github.com',
+    'https://news.ycombinator.com',
+  ];
+
+  const warmupSites = sites || defaultSites;
+
+  // Pick 1 random warmup site
+  const site = warmupSites[Math.floor(Math.random() * warmupSites.length)];
+
+  try {
+    await page.goto(site, { waitUntil: 'domcontentloaded', timeout: 10000 });
+    await randomDelay(1000, 3000);
+
+    if (scrollAfter) {
+      await humanScroll(page, { scrolls: 2 });
+    }
+
+    // Random mouse movement
+    await humanMouseMove(page, 300 + Math.random() * 600, 200 + Math.random() * 400);
+    await randomDelay(500, 1500);
+  } catch {
+    // Warmup failure is not critical
+  }
+}
+
+/**
+ * Post-navigation human behavior — run after page loads
+ */
+export async function postNavigationBehavior(page, opts = {}) {
+  const { scroll = true, mouseMove = true, minDelay = 500, maxDelay = 1500 } = opts;
+
+  // Wait a natural amount before interacting
+  await randomDelay(minDelay, maxDelay);
+
+  // Slight mouse movement (human looking at page)
+  if (mouseMove) {
+    await humanMouseMove(page, 300 + Math.random() * 600, 150 + Math.random() * 300);
+  }
+
+  // Natural scroll to show content loading
+  if (scroll) {
+    await humanScroll(page, { scrolls: 1 });
+  }
+}
+
+// --- Math helpers ---
+
+function cubicBezier(t, p0, p1, p2, p3) {
+  const mt = 1 - t;
+  return mt * mt * mt * p0 + 3 * mt * mt * t * p1 + 3 * mt * t * t * p2 + t * t * t * p3;
+}

package/src/index.js

@@ -0,0 +1,32 @@
+/**
+ * stealth-cli — Public API (SDK mode)
+ */
+
+// Core browser
+export {
+  launchBrowser, closeBrowser, navigate, getSnapshot,
+  getTextContent, getTitle, getUrl, takeScreenshot, evaluate, waitForReady,
+} from './browser.js';
+
+// Search
+export { expandMacro, getSupportedEngines } from './macros.js';
+export { getExtractorByEngine, getExtractorByUrl } from './extractors/index.js';
+
+// Cookies
+export { parseCookieFile, loadCookies } from './cookies.js';
+
+// Retry + Humanize
+export { withRetry, navigateWithRetry } from './retry.js';
+export {
+  randomDelay, humanScroll, humanMouseMove,
+  humanType, humanClick, warmup, postNavigationBehavior,
+} from './humanize.js';
+
+// Profile + Session + Proxy
+export { createProfile, loadProfile, listProfiles, deleteProfile } from './profiles.js';
+export { getSession, saveSession, captureSession, restoreSession, listSessions } from './session.js';
+export { addProxy, removeProxy, listProxies, getNextProxy, getRandomProxy, testProxy } from './proxy-pool.js';
+
+// Daemon
+export { isDaemonRunning } from './daemon.js';
+export { daemonStatus, daemonShutdown } from './client.js';

package/src/macros.js

@@ -0,0 +1,36 @@
+/**
+ * Search macros - URL templates for common search engines and sites
+ * Inspired by camofox-browser (MIT License)
+ */
+
+const MACROS = {
+  google: (q) => `https://www.google.com/search?q=${encodeURIComponent(q)}`,
+  youtube: (q) => `https://www.youtube.com/results?search_query=${encodeURIComponent(q)}`,
+  amazon: (q) => `https://www.amazon.com/s?k=${encodeURIComponent(q)}`,
+  reddit: (q) => `https://www.reddit.com/search?q=${encodeURIComponent(q)}`,
+  wikipedia: (q) => `https://en.wikipedia.org/wiki/Special:Search?search=${encodeURIComponent(q)}`,
+  twitter: (q) => `https://twitter.com/search?q=${encodeURIComponent(q)}`,
+  yelp: (q) => `https://www.yelp.com/search?find_desc=${encodeURIComponent(q)}`,
+  linkedin: (q) => `https://www.linkedin.com/search/results/all/?keywords=${encodeURIComponent(q)}`,
+  tiktok: (q) => `https://www.tiktok.com/search?q=${encodeURIComponent(q)}`,
+  github: (q) => `https://github.com/search?q=${encodeURIComponent(q)}&type=repositories`,
+  stackoverflow: (q) => `https://stackoverflow.com/search?q=${encodeURIComponent(q)}`,
+  npmjs: (q) => `https://www.npmjs.com/search?q=${encodeURIComponent(q)}`,
+  bing: (q) => `https://www.bing.com/search?q=${encodeURIComponent(q)}`,
+  duckduckgo: (q) => `https://duckduckgo.com/?q=${encodeURIComponent(q)}`,
+};
+
+/**
+ * Expand a search macro to a URL
+ */
+export function expandMacro(engine, query) {
+  const fn = MACROS[engine.toLowerCase()];
+  return fn ? fn(query) : null;
+}
+
+/**
+ * Get list of supported search engines
+ */
+export function getSupportedEngines() {
+  return Object.keys(MACROS);
+}