webpeel 0.21.53 → 0.21.56

package/dist/core/pipeline.js

@@ -1217,52 +1217,6 @@ export async function finalize(ctx) {
             log.error('Change tracking failed:', error);
         }
     }
-    // ── Auto-escalation: retry thin content with browser rendering ──────────────
-    // If simple fetch returned very little content and user didn't explicitly disable render,
-    // automatically retry with browser rendering to handle JS-heavy/paywalled sites.
-    const preEscalationWords = ctx.content.trim().split(/\s+/).filter((w) => w.length > 0).length;
-    const escalationFetchMethod = fetchResult?.method || 'unknown';
-    const alreadyTriedBrowser = escalationFetchMethod === 'browser' || escalationFetchMethod === 'stealth'
-        || options.render || options.stealth;
-    const userDisabledRender = options.render === false;
-    const escalationCandidate = preEscalationWords < 200 && preEscalationWords > 0
-        && escalationFetchMethod === 'simple' && !alreadyTriedBrowser && !userDisabledRender
-        && !ctx._escalated;
-    if (escalationCandidate) {
-        log.info(`thin content (${preEscalationWords}w) from simple fetch, auto-escalating to browser render for ${ctx.url}`);
-        ctx._escalated = true;
-        try {
-            const { smartFetch } = await import('./strategies.js');
-            const browserResult = await smartFetch(ctx.url, {
-                forceBrowser: true,
-                stealth: false,
-                timeoutMs: options.timeout || 15000,
-                proxy: options.proxy,
-            });
-            if (browserResult.html && browserResult.html.length > (fetchResult?.html?.length || 0)) {
-                const { htmlToMarkdown } = await import('./markdown.js');
-                const browserContent = htmlToMarkdown(browserResult.html);
-                const browserWords = browserContent.trim().split(/\s+/).filter((w) => w.length > 0).length;
-                if (browserWords > preEscalationWords) {
-                    log.info(`browser escalation improved content: ${preEscalationWords}w → ${browserWords}w`);
-                    ctx.content = browserContent;
-                    ctx.fetchResult = browserResult;
-                    ctx.fetchResult.method = 'browser-escalation';
-                }
-                else {
-                    log.debug(`browser escalation did not improve (${browserWords}w vs ${preEscalationWords}w)`);
-                }
-                // Always clean up browser resources
-                if (browserResult.page)
-                    await browserResult.page.close().catch(() => { });
-                if (browserResult.browser)
-                    await browserResult.browser.close().catch(() => { });
-            }
-        }
-        catch (e) {
-            log.debug('browser escalation failed:', e instanceof Error ? e.message : e);
-        }
-    }
     // Generate AI summary if requested
     if (options.summary && options.llm) {
         try {

package/dist/core/source-credibility.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Source credibility scoring — lightweight, zero dependencies.
+ *
+ * Classifies URLs by trustworthiness:
+ *   - Official (★★★): .gov, .edu, .mil, WHO, NIH, academic journals
+ *   - Verified (★★):  Wikipedia, Reuters, BBC, GitHub, StackOverflow
+ *   - General (★):    Everything else
+ */
+export interface SourceCredibility {
+    tier: 'official' | 'verified' | 'general';
+    stars: number;
+    label: string;
+}
+/**
+ * Assess the credibility of a source URL.
+ */
+export declare function getSourceCredibility(url: string): SourceCredibility;

package/dist/core/source-credibility.js

@@ -0,0 +1,83 @@
+/**
+ * Source credibility scoring — lightweight, zero dependencies.
+ *
+ * Classifies URLs by trustworthiness:
+ *   - Official (★★★): .gov, .edu, .mil, WHO, NIH, academic journals
+ *   - Verified (★★):  Wikipedia, Reuters, BBC, GitHub, StackOverflow
+ *   - General (★):    Everything else
+ */
+/** Official TLDs and hostnames that indicate high-authority sources */
+const OFFICIAL_TLDS = new Set(['.gov', '.edu', '.mil']);
+const OFFICIAL_HOSTNAMES = new Set([
+    // Academic / research
+    'arxiv.org', 'scholar.google.com', 'pubmed.ncbi.nlm.nih.gov', 'ncbi.nlm.nih.gov',
+    'jstor.org', 'nature.com', 'science.org', 'cell.com', 'nejm.org', 'bmj.com',
+    'thelancet.com', 'plos.org', 'springer.com', 'elsevier.com',
+    // International organisations
+    'who.int', 'un.org', 'worldbank.org', 'imf.org', 'oecd.org', 'europa.eu',
+    // Official tech documentation
+    'docs.python.org', 'developer.mozilla.org', 'nodejs.org', 'rust-lang.org',
+    'docs.microsoft.com', 'learn.microsoft.com', 'developer.apple.com',
+    'developer.android.com', 'php.net', 'ruby-lang.org', 'golang.org', 'go.dev',
+    // Health / medicine
+    'cdc.gov', 'nih.gov', 'fda.gov', 'mayoclinic.org', 'clevelandclinic.org',
+    'webmd.com', 'medlineplus.gov',
+    // Standards / specs
+    'w3.org', 'ietf.org', 'rfc-editor.org', 'iso.org',
+]);
+const VERIFIED_HOSTNAMES = new Set([
+    // Encyclopaedia / reference
+    'wikipedia.org', 'en.wikipedia.org', 'britannica.com',
+    // Reputable news agencies
+    'reuters.com', 'apnews.com', 'bbc.com', 'bbc.co.uk', 'nytimes.com',
+    'washingtonpost.com', 'theguardian.com', 'economist.com', 'ft.com',
+    'cnn.com', 'npr.org', 'pbs.org',
+    // Developer resources
+    'github.com', 'stackoverflow.com', 'npmjs.com', 'pypi.org',
+    'crates.io', 'docs.rs', 'packagist.org', 'rubygems.org',
+    // Official cloud / vendor docs
+    'docs.aws.amazon.com', 'cloud.google.com', 'docs.github.com',
+    'azure.microsoft.com', 'registry.terraform.io',
+    // Reputable tech publications
+    'arstechnica.com', 'wired.com', 'techcrunch.com', 'theverge.com',
+    // National Geographic, Smithsonian
+    'nationalgeographic.com', 'smithsonianmag.com',
+]);
+/**
+ * Assess the credibility of a source URL.
+ */
+export function getSourceCredibility(url) {
+    try {
+        const hostname = new URL(url).hostname.toLowerCase().replace(/^www\./, '');
+        // Check official TLDs
+        for (const tld of OFFICIAL_TLDS) {
+            if (hostname.endsWith(tld)) {
+                return { tier: 'official', stars: 3, label: 'OFFICIAL SOURCE' };
+            }
+        }
+        // Check known official hostnames  
+        if (OFFICIAL_HOSTNAMES.has(hostname)) {
+            return { tier: 'official', stars: 3, label: 'OFFICIAL SOURCE' };
+        }
+        // Check parent domain (e.g. en.wikipedia.org → wikipedia.org)
+        const parts = hostname.split('.');
+        if (parts.length > 2) {
+            const parentDomain = parts.slice(-2).join('.');
+            if (OFFICIAL_HOSTNAMES.has(parentDomain)) {
+                return { tier: 'official', stars: 3, label: 'OFFICIAL SOURCE' };
+            }
+            if (VERIFIED_HOSTNAMES.has(parentDomain)) {
+                return { tier: 'verified', stars: 2, label: 'VERIFIED' };
+            }
+        }
+        // Check known verified hostnames
+        if (VERIFIED_HOSTNAMES.has(hostname)) {
+            return { tier: 'verified', stars: 2, label: 'VERIFIED' };
+        }
+        // Everything else
+        return { tier: 'general', stars: 1, label: 'UNVERIFIED' };
+    }
+    catch {
+        return { tier: 'general', stars: 1, label: 'UNVERIFIED' };
+    }
+}

package/dist/server/routes/search.js

@@ -9,6 +9,7 @@ import { peel } from '../../index.js';
 import { simpleFetch } from '../../core/fetcher.js';
 import { searchCache } from '../../core/fetch-cache.js';
 import { getSearchProvider, getBestSearchProvider, } from '../../core/search-provider.js';
+import { getSourceCredibility } from '../../core/source-credibility.js';
 export function createSearchRouter(authStore) {
     const router = Router();
     // LRU cache: 15 minute TTL, max 500 entries, 50MB total size
@@ -210,6 +211,19 @@ export function createSearchRouter(authStore) {
                         }
                     }
                 }
+                // Add credibility scores and sort by trustworthiness
+                const tierOrder = { official: 0, verified: 1, general: 2 };
+                results = results
+                    .map(r => {
+                    const cred = getSourceCredibility(r.url);
+                    return { ...r, credibility: cred };
+                })
+                    .sort((a, b) => {
+                    const aTier = tierOrder[a.credibility?.tier || 'general'] ?? 2;
+                    const bTier = tierOrder[b.credibility?.tier || 'general'] ?? 2;
+                    return aTier - bTier; // Official first, then verified, then general
+                })
+                    .map((r, i) => ({ ...r, rank: i + 1 }));
                 data.web = results;
             }
             // Fetch news results (DDG only — Brave news is not supported via HTML scraping)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.53",
+  "version": "0.21.56",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",