webpeel 0.21.18 → 0.21.20

package/dist/core/domain-extractors.js

@@ -153,7 +153,12 @@ async function fetchJson(url, customHeaders) {
         Accept: 'application/json',
         ...customHeaders,
     });
-    return tryParseJson(result.html);
+    const parsed = tryParseJson(result.html);
+    // Debug: log GitHub API failures to help diagnose rate limiting issues
+    if (!parsed && url.includes('api.github.com')) {
+        console.error(`[github-debug] fetchJson failed for ${url} — raw response (first 200): ${result.html?.substring(0, 200)}`);
+    }
+    return parsed;
 }
 /** Fetch JSON with exponential backoff retry on 429 / rate-limit errors. */
 async function fetchJsonWithRetry(url, headers, retries = 2, baseDelayMs = 1000) {
@@ -780,8 +785,13 @@ async function githubExtractor(_html, url) {
     const ghHeaders = { Accept: 'application/vnd.github.v3+json' };
     // Use GITHUB_TOKEN if available for higher rate limits (5000/hr vs 60/hr)
     const ghToken = process.env.GITHUB_TOKEN || process.env.GH_TOKEN;
-    if (ghToken)
+    if (ghToken) {
         ghHeaders.Authorization = `token ${ghToken}`;
+        console.log(`[github-debug] Using token (prefix: ${ghToken.substring(0, 8)}..., len: ${ghToken.length})`);
+    }
+    else {
+        console.warn('[github-debug] No GITHUB_TOKEN found — using anonymous (60/hr limit)');
+    }
     // User profile: /username (single segment)
     if (pathParts.length === 1) {
         const username = pathParts[0];
@@ -908,12 +918,14 @@ ${commentsMd || '*No comments.*'}`;
     }
     // Repository page: /owner/repo (and no deeper path we handle above)
     if (pathParts.length >= 2) {
-        const [repoData, readmeData] = await Promise.all([
-            fetchJson(`https://api.github.com/repos/${owner}/${repo}`, ghHeaders),
-            fetchJson(`https://api.github.com/repos/${owner}/${repo}/readme`, ghHeaders).catch(() => null),
-        ]);
+        // Sequential fetches to avoid secondary rate limits on popular repos
+        const repoData = await fetchJsonWithRetry(`https://api.github.com/repos/${owner}/${repo}`, ghHeaders, 2, 1000);
         if (!repoData || repoData.message === 'Not Found')
             return null;
+        // Secondary rate limit check
+        if (repoData.message?.includes('secondary rate limit') || repoData.message?.includes('abuse'))
+            return null;
+        const readmeData = await fetchJsonWithRetry(`https://api.github.com/repos/${owner}/${repo}/readme`, ghHeaders, 1, 500).catch(() => null);
         // README content is base64 encoded
         let readmeText = '';
         if (readmeData?.content) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.18",
+  "version": "0.21.20",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",