webpeel 0.21.9 → 0.21.11

package/dist/cli/commands/fetch.js

@@ -409,6 +409,28 @@ export async function runFetch(url, options) {
     }
     // Suppress spinner when --progress is active (progress lines replace it)
     const spinner = (options.silent || options.progress) ? null : ora('Fetching...').start();
+    // Auto progress: after 3 s, update spinner text with elapsed time + method hints
+    // Updated every 2 s so the user knows we're still working.
+    const autoProgressStart = Date.now();
+    const autoProgressSteps = [
+        { afterMs: 3000, text: '⏳ Fetching... (slow response)' },
+        { afterMs: 6000, text: '⏳ Fetching with browser... ({s}s)' },
+        { afterMs: 12000, text: '⏳ Fetching with browser... ({s}s — stealth may be needed)' },
+        { afterMs: 20000, text: '⏳ Fetching with stealth browser + proxy... ({s}s)' },
+    ];
+    let autoProgressStepIdx = 0;
+    const autoProgressInterval = spinner ? setInterval(() => {
+        const elapsed = Date.now() - autoProgressStart;
+        const secs = Math.round(elapsed / 1000);
+        while (autoProgressStepIdx < autoProgressSteps.length &&
+            elapsed >= autoProgressSteps[autoProgressStepIdx].afterMs) {
+            autoProgressStepIdx++;
+        }
+        if (autoProgressStepIdx > 0 && spinner) {
+            const tmpl = autoProgressSteps[autoProgressStepIdx - 1].text;
+            spinner.text = tmpl.replace('{s}', String(secs));
+        }
+    }, 2000) : null;
     try {
         // Validate options
         if (options.wait && (options.wait < 0 || options.wait > 60000)) {
@@ -661,11 +683,13 @@ export async function runFetch(url, options) {
         if (resolvedProfileName) {
             touchProfile(resolvedProfileName);
         }
-        // Stop progress interval and show final result
+        // Stop progress intervals and show final result
         if (progressInterval) {
             clearInterval(progressInterval);
             progressInterval = undefined;
         }
+        if (autoProgressInterval)
+            clearInterval(autoProgressInterval);
         if (options.progress) {
             const method = result.method || 'simple';
             const elapsedSec = ((result.elapsed || (Date.now() - progressStart)) / 1000).toFixed(1);
@@ -1067,6 +1091,8 @@ export async function runFetch(url, options) {
         process.exit(0);
     }
     catch (error) {
+        if (autoProgressInterval)
+            clearInterval(autoProgressInterval);
         if (spinner) {
             spinner.fail('Failed to fetch');
         }

package/dist/core/domain-extractors.js

@@ -308,6 +308,7 @@ async function twitterExtractor(html, url) {
             if (fxData && fxData.code === 200 && fxData.user) {
                 const u = fxData.user;
                 const structured = {
+                    title: `${u.name || ''} (@${u.screen_name || ''}) on X/Twitter`,
                     name: u.name || '',
                     handle: '@' + (u.screen_name || ''),
                     bio: u.description || '',
@@ -1500,6 +1501,7 @@ async function npmExtractor(_html, url) {
         }
         catch { /* optional */ }
         const structured = {
+            title: `${data.name}@${latest || 'unknown'}`,
             name: data.name,
             description: data.description || '',
             version: latest || 'unknown',

package/dist/core/search-provider.d.ts

@@ -17,6 +17,8 @@ export interface WebSearchResult {
     title: string;
     url: string;
     snippet: string;
+    /** Relevance score (0–1) based on keyword overlap with query. Added by filterRelevantResults. */
+    relevanceScore?: number;
 }
 export interface WebSearchOptions {
     /** Number of results (1-10) */
@@ -71,6 +73,18 @@ declare class ProviderStatsTracker {
  * (e.g. in tests) and to log diagnostics.
  */
 export declare const providerStats: ProviderStatsTracker;
+/**
+ * Filter and rank results by relevance to the original query.
+ *
+ * 1. Extract meaningful keywords from the query (remove stop words).
+ * 2. Score each result by keyword overlap with title + URL + snippet.
+ * 3. Remove results with zero overlap (completely irrelevant).
+ * 4. Sort descending by score, keeping original index as tiebreaker.
+ * 5. Attach `relevanceScore` (0–1) to each surviving result.
+ *
+ * Results without any scores (query produced no keywords) are returned as-is.
+ */
+export declare function filterRelevantResults(results: WebSearchResult[], query: string): WebSearchResult[];
 /**
  * StealthSearchProvider — self-hosted multi-engine search
  *
@@ -115,6 +129,11 @@ export declare class DuckDuckGoProvider implements SearchProvider {
      */
     private searchLite;
     searchWeb(query: string, options: WebSearchOptions): Promise<WebSearchResult[]>;
+    /**
+     * Exposed for testing: score and filter a pre-fetched result list against a query.
+     * Equivalent to calling filterRelevantResults() directly.
+     */
+    filterResults(results: WebSearchResult[], query: string): WebSearchResult[];
 }
 export declare class BraveSearchProvider implements SearchProvider {
     readonly id: SearchProviderId;

package/dist/core/search-provider.js

@@ -195,6 +195,91 @@ function normalizeUrlForDedupe(rawUrl) {
             .replace(/\/+$/g, '');
     }
 }
+// ============================================================
+// Result Relevance Filtering
+// Lightweight keyword-overlap scoring — no external deps.
+// Applied after fetching raw results to remove completely off-
+// topic hits (e.g., a grammar article returned for "used cars").
+// ============================================================
+const STOP_WORDS = new Set([
+    'the', 'a', 'an', 'is', 'are', 'was', 'were', 'in', 'on', 'at', 'to', 'for',
+    'of', 'with', 'how', 'what', 'where', 'when', 'why', 'best', 'top', 'most',
+    'and', 'or', 'but', 'not', 'do', 'does', 'did', 'be', 'been', 'have', 'has',
+    'buy', 'get', 'find', 'about', 'from', 'by', 'its', 'it', 'this', 'that',
+]);
+/**
+ * Extract meaningful keywords from a search query by stripping stop words and
+ * short tokens.  Returns lowercase tokens, deduped.
+ */
+function extractKeywords(query) {
+    const seen = new Set();
+    return query
+        .toLowerCase()
+        .replace(/[^a-z0-9\s]/g, ' ')
+        .split(/\s+/)
+        .filter(w => w.length >= 2 && !STOP_WORDS.has(w))
+        .filter(w => {
+        if (seen.has(w))
+            return false;
+        seen.add(w);
+        return true;
+    });
+}
+/**
+ * Compute a [0, 1] relevance score for a single result against extracted keywords.
+ * Weights: title 0.5, URL 0.3, snippet 0.2.
+ */
+function scoreResult(result, keywords) {
+    if (keywords.length === 0)
+        return 1;
+    const titleLower = (result.title || '').toLowerCase();
+    const urlLower = (result.url || '').toLowerCase();
+    const snippetLower = (result.snippet || '').toLowerCase();
+    let titleHits = 0;
+    let urlHits = 0;
+    let snippetHits = 0;
+    for (const kw of keywords) {
+        if (titleLower.includes(kw))
+            titleHits++;
+        if (urlLower.includes(kw))
+            urlHits++;
+        if (snippetLower.includes(kw))
+            snippetHits++;
+    }
+    const titleScore = titleHits / keywords.length;
+    const urlScore = urlHits / keywords.length;
+    const snippetScore = snippetHits / keywords.length;
+    return titleScore * 0.5 + urlScore * 0.3 + snippetScore * 0.2;
+}
+/**
+ * Filter and rank results by relevance to the original query.
+ *
+ * 1. Extract meaningful keywords from the query (remove stop words).
+ * 2. Score each result by keyword overlap with title + URL + snippet.
+ * 3. Remove results with zero overlap (completely irrelevant).
+ * 4. Sort descending by score, keeping original index as tiebreaker.
+ * 5. Attach `relevanceScore` (0–1) to each surviving result.
+ *
+ * Results without any scores (query produced no keywords) are returned as-is.
+ */
+export function filterRelevantResults(results, query) {
+    const keywords = extractKeywords(query);
+    if (keywords.length === 0)
+        return results; // no keywords to filter on
+    const scored = results.map((r, idx) => ({
+        result: r,
+        score: scoreResult(r, keywords),
+        idx,
+    }));
+    // Drop results with zero overlap
+    const relevant = scored.filter(s => s.score > 0);
+    // Sort by score descending, original order as tiebreaker
+    relevant.sort((a, b) => (b.score !== a.score ? b.score - a.score : a.idx - b.idx));
+    return relevant.map(s => ({
+        ...s.result,
+        relevanceScore: Math.min(1, s.score),
+    }));
+}
 /**
  * StealthSearchProvider — self-hosted multi-engine search
  *
@@ -470,7 +555,10 @@ export class StealthSearchProvider {
             if (deduped.length >= count)
                 break;
         }
-        return deduped;
+        // Relevance filtering: remove completely off-topic results, score the rest
+        const filtered = filterRelevantResults(deduped, query);
+        // Respect the original count limit after filtering
+        return filtered.slice(0, count);
     }
 }
 export class DuckDuckGoProvider {
@@ -688,7 +776,9 @@ export class DuckDuckGoProvider {
                         providerStats.record('ddg-http', true);
                         log.debug(`source=ddg-http returned ${results.length} results` +
                             (ddgTimeoutMs < 8_000 ? ` (fast-timeout ${ddgTimeoutMs}ms)` : ''));
-                        return results;
+                        // Apply relevance filtering before returning
+                        const filtered = filterRelevantResults(results, query);
+                        return filtered.length > 0 ? filtered : results; // fallback to unfiltered if all removed
                     }
                     ddgSucceeded = true; // connected OK, just 0 results
                 }
@@ -721,7 +811,9 @@ export class DuckDuckGoProvider {
                     providerStats.record('ddg-lite', true);
                     log.debug(`source=ddg-lite returned ${liteResults.length} results` +
                         (liteTimeoutMs < 8_000 ? ` (fast-timeout ${liteTimeoutMs}ms)` : ''));
-                    return liteResults;
+                    // Apply relevance filtering before returning
+                    const filteredLite = filterRelevantResults(liteResults, query);
+                    return filteredLite.length > 0 ? filteredLite : liteResults;
                 }
                 providerStats.record('ddg-lite', false);
                 log.debug('DDG Lite also returned 0 results');
@@ -756,6 +848,7 @@ export class DuckDuckGoProvider {
         log.debug('Trying stealth browser search (DDG + Bing + Ecosia)...');
         try {
             const stealthProvider = new StealthSearchProvider();
+            // StealthSearchProvider already applies filterRelevantResults internally.
             const stealthResults = await stealthProvider.searchWeb(query, options);
             if (stealthResults.length > 0) {
                 log.debug(`source=stealth returned ${stealthResults.length} results`);
@@ -768,6 +861,13 @@ export class DuckDuckGoProvider {
         }
         return [];
     }
+    /**
+     * Exposed for testing: score and filter a pre-fetched result list against a query.
+     * Equivalent to calling filterRelevantResults() directly.
+     */
+    filterResults(results, query) {
+        return filterRelevantResults(results, query);
+    }
 }
 export class BraveSearchProvider {
     id = 'brave';

package/dist/core/structured-extract.js

@@ -296,7 +296,13 @@ function heuristicExtractNumber(fieldName, content) {
     }
     // Year
     if (/year/.test(lf)) {
-        const m = content.match(/\b(20\d{2})\b/);
+        // Match 4-digit years (1900-2099), prefer explicit "Year: YYYY" pattern first
+        const explicit = content.match(/\bYear[:\s]+(\d{4})\b/i);
+        if (explicit?.[1]) {
+            const n = parseInt(explicit[1]);
+            return isNaN(n) ? null : n;
+        }
+        const m = content.match(/\b((?:19|20)\d{2})\b/);
         if (m?.[1]) {
             const n = parseInt(m[1]);
             return isNaN(n) ? null : n;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.9",
+  "version": "0.21.11",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",