webpeel 0.21.13 → 0.21.15

package/dist/core/domain-extractors.js

@@ -778,6 +778,10 @@ async function githubExtractor(_html, url) {
     if (pathParts.length === 0)
         return null;
     const ghHeaders = { Accept: 'application/vnd.github.v3+json' };
+    // Use GITHUB_TOKEN if available for higher rate limits (5000/hr vs 60/hr)
+    const ghToken = process.env.GITHUB_TOKEN || process.env.GH_TOKEN;
+    if (ghToken)
+        ghHeaders.Authorization = `token ${ghToken}`;
     // User profile: /username (single segment)
     if (pathParts.length === 1) {
         const username = pathParts[0];
@@ -986,6 +990,27 @@ async function hackerNewsExtractor(_html, url) {
         const storyData = await fetchJson(`https://hacker-news.firebaseio.com/v0/item/${itemId}.json`);
         if (!storyData)
             return null;
+        // Comment items — fetch parent story for context
+        if (storyData.type === 'comment') {
+            const parentId = storyData.parent;
+            let parentTitle = '';
+            if (parentId) {
+                try {
+                    const parentData = await fetchJson(`https://hacker-news.firebaseio.com/v0/item/${parentId}.json`);
+                    parentTitle = parentData?.title || '';
+                    // Walk up to root story if parent is also a comment
+                    if (!parentTitle && parentData?.parent) {
+                        const rootData = await fetchJson(`https://hacker-news.firebaseio.com/v0/item/${parentData.parent}.json`);
+                        parentTitle = rootData?.title || '';
+                    }
+                }
+                catch { /* non-fatal */ }
+            }
+            const text = storyData.text ? stripHtml(storyData.text) : '';
+            const titleStr = parentTitle ? `Comment on: ${parentTitle}` : 'HN Comment';
+            const cleanContent = `## 🟠 ${titleStr}\n\n**Author:** ${storyData.by || '[deleted]'} | **Posted:** ${unixToIso(storyData.time)}\n\n${text}`;
+            return { domain: 'news.ycombinator.com', type: 'comment', structured: { title: titleStr, author: storyData.by, text }, cleanContent };
+        }
         const type = storyData.type === 'story' ? 'story' :
             storyData.type === 'ask' ? 'ask_hn' :
                 storyData.type === 'show' ? 'show_hn' :

package/dist/core/pipeline.js

@@ -278,15 +278,9 @@ export async function fetchContent(ctx) {
                     ctx.rawHtmlSize = ddResult.rawHtmlSize;
                 }
                 else {
-                    // For API-first extractors (HN, Reddit, GitHub), estimate what the raw page
-                    // would cost by doing a quick HEAD request for Content-Length
-                    try {
-                        const headResp = await fetch(ctx.url, { method: 'HEAD', signal: AbortSignal.timeout(2000) });
-                        const cl = headResp.headers.get('content-length');
-                        if (cl)
-                            ctx.rawHtmlSize = parseInt(cl, 10);
-                    }
-                    catch { /* non-fatal — token estimate will be undefined */ }
+                    // For API-first extractors (HN, Reddit, GitHub), the raw HTML page is typically
+                    // 6-10x larger than the extracted content. Estimate conservatively at 7x.
+                    ctx.rawHtmlSize = ddResult.cleanContent.length * 7;
                 }
                 // Create minimal fetchResult so downstream stages don't crash
                 ctx.fetchResult = {
@@ -360,6 +354,10 @@ export async function fetchContent(ctx) {
                     if (ddResult.rawHtmlSize && ddResult.rawHtmlSize > 0) {
                         ctx.rawHtmlSize = ddResult.rawHtmlSize;
                     }
+                    else {
+                        // Estimate raw HTML size for API-first extractors (7x compression factor)
+                        ctx.rawHtmlSize = ddResult.cleanContent.length * 7;
+                    }
                     ctx.fetchResult = {
                         html: ddResult.cleanContent,
                         url: ctx.url,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.13",
+  "version": "0.21.15",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",