pi-smart-fetch 0.2.28 → 0.2.30

package/dist/index.js

@@ -8,8 +8,8 @@ import { readFile, mkdir, chmod, writeFile, unlink } from 'fs/promises';
 import { tmpdir } from 'os';
 import { join, parse } from 'path';
 import { pipeline } from 'stream/promises';
-import { getProfiles, fetch } from '@thinkscape/wreq-js';
 import { Defuddle } from 'defuddle/node';
+import { getProfiles, fetch } from 'wreq-js';
 import { parseHTML } from 'linkedom';
 
 var __create = Object.create;
@@ -9609,6 +9609,12 @@ function buildPlainTextResult(opts, finalUrl, rawBody, format, maxChars, browser
     os
   };
 }
+function isTwitterJsDisabledPage(document, url) {
+  if (!/^(https?:\/\/)?(www\.)?(x\.com|twitter\.com)\//i.test(url))
+    return false;
+  const text = document.body?.textContent ?? document.documentElement?.textContent ?? "";
+  return text.includes("JavaScript is disabled") && text.includes("supported browser");
+}
 function extractDomTextFallback(document) {
   const bodyText = document.body?.textContent ?? document.documentElement?.textContent ?? "";
   return bodyText.replace(/\r\n/g, "\n").replace(/\n{3,}/g, "\n\n").split("\n").map((line) => line.trim()).join("\n").replace(/[ \t]{2,}/g, " ").trim();
@@ -10199,15 +10205,47 @@ function createDefuddleFetch(dependencies = runtimeDependencies) {
       });
       const fallbackDocument = parseLinkedomHTML(rawBody, finalUrl);
       const extractionDocument = parseLinkedomHTML(rawBody, finalUrl);
-      const extracted = await dependencies.defuddle(
-        extractionDocument,
-        finalUrl,
-        {
-          markdown: format !== "html",
-          removeImages,
-          includeReplies
+      if (isTwitterJsDisabledPage(fallbackDocument, opts.url)) {
+        return {
+          error: `Server returned HTTP 404 Not Found for ${opts.url}.`,
+          code: "http_error",
+          phase: "loading",
+          retryable: false,
+          timeoutMs,
+          url: opts.url,
+          finalUrl,
+          statusCode: 404,
+          statusText: "Not Found",
+          mimeType: normalizeContentType(contentType) || void 0,
+          contentLength: errorContext.contentLength
+        };
+      }
+      let extracted;
+      try {
+        const origConsoleError = console.error;
+        const suppressedErrors = [];
+        console.error = (...args) => {
+          suppressedErrors.push(args);
+        };
+        try {
+          extracted = await dependencies.defuddle(
+            extractionDocument,
+            finalUrl,
+            {
+              markdown: format !== "html",
+              removeImages,
+              includeReplies
+            }
+          );
+        } finally {
+          console.error = origConsoleError;
         }
-      );
+      } catch (_error) {
+        extracted = {
+          content: void 0,
+          wordCount: 0
+        };
+      }
       let extractedContent = extracted.content;
       let wordCount = extracted.wordCount;
       if (!extractedContent || wordCount === 0) {