web-to-markdown 0.1.0 → 0.1.1

package/dist/bin/web-to-markdown.js

@@ -1,8 +1,8 @@
 #!/usr/bin/env node
 
 // src/cli.ts
-import { resolve, dirname } from "path";
-import { existsSync, lstatSync, readFileSync } from "fs";
+import { resolve, dirname, sep } from "path";
+import { existsSync, readFileSync } from "fs";
 import { realpath, writeFile } from "fs/promises";
 import { fileURLToPath } from "url";
 import { Command } from "commander";
@@ -47,10 +47,10 @@ function expandIPv6(ip) {
   } else {
     groups = ip.split(":");
   }
-  return groups.map((g) => g.toLowerCase()).join(":");
+  return groups.map((g) => g.toLowerCase().padStart(4, "0")).join(":");
 }
 function normalizeIP(hostname) {
-  const bare = hostname.replace(/^\[|\]$/g, "");
+  const bare = hostname.replace(/^\[|\]$/g, "").replace(/%.*$/, "");
   if (isIP(bare) === 4) return bare;
   if (isIP(bare) === 6) {
     const ffmpDotted = bare.match(/^::ffff:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/i);
@@ -62,7 +62,9 @@ function normalizeIP(hostname) {
       return `${high >> 8 & 255}.${high & 255}.${low >> 8 & 255}.${low & 255}`;
     }
     const fullFfmpHex = expandIPv6(bare);
-    const fullFfmpMatch = fullFfmpHex.match(/^0:0:0:0:0:ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/i);
+    const fullFfmpMatch = fullFfmpHex.match(
+      /^0000:0000:0000:0000:0000:ffff:([0-9a-f]{4}):([0-9a-f]{4})$/i
+    );
     if (fullFfmpMatch) {
       const high = parseInt(fullFfmpMatch[1], 16);
       const low = parseInt(fullFfmpMatch[2], 16);
@@ -119,20 +121,21 @@ function isPrivateIP(ip) {
   if (ip.startsWith("198.51.100.")) return true;
   if (ip.startsWith("203.0.113.")) return true;
   if (/^198\.(18|19)\./.test(ip)) return true;
-  if (/^24[0-9]\./.test(ip) || ip.startsWith("255.")) return true;
+  if (ip.includes(".") && !ip.includes(":")) {
+    const firstOctet = parseInt(ip.split(".")[0], 10);
+    if (firstOctet >= 224) return true;
+  }
   const expanded = ip.includes("::") ? expandIPv6(ip) : ip.toLowerCase();
-  if (ip === "::1" || ip === "::" || expanded === "0:0:0:0:0:0:0:1" || expanded === "0:0:0:0:0:0:0:0")
+  if (ip === "::1" || ip === "::" || expanded === "0000:0000:0000:0000:0000:0000:0000:0001" || expanded === "0000:0000:0000:0000:0000:0000:0000:0000")
     return true;
   if (expanded.startsWith("fc") || expanded.startsWith("fd")) return true;
-  if (expanded.startsWith("fe80:") || expanded.startsWith("fe8") || expanded.startsWith("fe9") || expanded.startsWith("fea") || expanded.startsWith("feb"))
-    return true;
-  if (expanded.startsWith("fec0:") || expanded.startsWith("fec") || expanded.startsWith("fed") || expanded.startsWith("fee") || expanded.startsWith("fef"))
-    return true;
+  if (/^fe[89ab][0-9a-f]:/.test(expanded)) return true;
+  if (/^fe[cdef][0-9a-f]:/.test(expanded)) return true;
   if (expanded.startsWith("ff")) return true;
-  if (expanded.startsWith("64:ff9b:")) return true;
+  if (expanded.startsWith("0064:ff9b:")) return true;
   if (expanded.startsWith("2002:")) return true;
-  if (expanded.startsWith("2001:db8:")) return true;
-  if (expanded.startsWith("2001:0:")) return true;
+  if (expanded.startsWith("2001:0db8:")) return true;
+  if (expanded.startsWith("2001:0000:")) return true;
   return false;
 }
 
@@ -182,6 +185,8 @@ var BLOCKED_HOSTNAME_SUFFIXES = [
   ".internal",
   ".local",
   ".localhost",
+  ".localdomain",
+  ".intranet",
   ".corp",
   ".home",
   ".lan"
@@ -202,13 +207,18 @@ async function resolveAndValidateHostname(hostname) {
     return normalizedIP;
   }
   try {
-    const { address } = await lookup(hostname);
-    if (isPrivateIP(address)) {
-      throw new SSRFError(
-        `Blocked request to "${hostname}" \u2014 resolves to private IP. Requests to internal networks are not allowed.`
-      );
+    const results = await lookup(hostname, { all: true });
+    if (results.length === 0) {
+      throw new NetworkError(`DNS lookup for "${hostname}" returned no results. Check the URL.`);
+    }
+    for (const { address } of results) {
+      if (isPrivateIP(address)) {
+        throw new SSRFError(
+          `Blocked request to "${hostname}" \u2014 resolves to private IP. Requests to internal networks are not allowed.`
+        );
+      }
     }
-    return address;
+    return results[0].address;
   } catch (err) {
     if (err instanceof SSRFError) throw err;
     if (err instanceof Error) {
@@ -233,7 +243,7 @@ async function validateUrl(url) {
       `Unsupported protocol "${parsed.protocol}". Only HTTP and HTTPS are supported.`
     );
   }
-  const hostname = parsed.hostname.toLowerCase();
+  const hostname = parsed.hostname.toLowerCase().replace(/\.$/, "");
   if (BLOCKED_HOSTNAMES.has(hostname)) {
     throw new SSRFError(
       `Blocked request to "${hostname}". Requests to internal hosts are not allowed.`
@@ -248,6 +258,20 @@ async function validateUrl(url) {
   }
   return resolveAndValidateHostname(hostname);
 }
+function extractCharset(headers) {
+  const raw = headers["content-type"];
+  const contentType = typeof raw === "string" ? raw : "";
+  const match = contentType.match(/charset\s*=\s*["']?([^"';,\s]+)["']?/i);
+  return match ? match[1].toLowerCase() : null;
+}
+function decodeBody(buffer, charset) {
+  const encoding = charset || "utf-8";
+  try {
+    return new TextDecoder(encoding).decode(buffer);
+  } catch {
+    return buffer.toString("utf-8");
+  }
+}
 var HTML_CONTENT_TYPES = [
   "text/html",
   "application/xhtml+xml",
@@ -282,11 +306,14 @@ function pinnedRequest(url, resolvedIP, timeout) {
     const parsed = new URL(url);
     const isHttps = parsed.protocol === "https:";
     const requestFn = isHttps ? httpsRequest : httpRequest;
+    const abortController = new AbortController();
+    const totalTimer = setTimeout(() => abortController.abort(), timeout);
     const req = requestFn(
       url,
       {
         method: "GET",
         timeout,
+        signal: abortController.signal,
         lookup: createPinnedLookup(resolvedIP),
         headers: {
           "User-Agent": "Mozilla/5.0 (compatible; web-to-markdown/0.1; +https://github.com/nidhi-singh02/mark-it-down)",
@@ -300,6 +327,7 @@ function pinnedRequest(url, resolvedIP, timeout) {
         res.on("data", (chunk) => {
           totalBytes += chunk.length;
           if (totalBytes > MAX_RESPONSE_SIZE) {
+            clearTimeout(totalTimer);
             req.destroy();
             reject(
               new ContentError(
@@ -311,24 +339,36 @@ function pinnedRequest(url, resolvedIP, timeout) {
           chunks.push(chunk);
         });
         res.on("end", () => {
+          clearTimeout(totalTimer);
           const bodyBuffer = Buffer.concat(chunks);
+          const responseHeaders = res.headers;
+          const charset = extractCharset(responseHeaders);
           resolve2({
             status: res.statusCode || 0,
-            headers: res.headers,
-            body: bodyBuffer.toString("utf-8"),
+            headers: responseHeaders,
+            body: decodeBody(bodyBuffer, charset),
             bodyBuffer,
             responseUrl: url
           });
         });
-        res.on("error", reject);
+        res.on("error", (err) => {
+          clearTimeout(totalTimer);
+          reject(err);
+        });
       }
     );
     req.on("timeout", () => {
+      clearTimeout(totalTimer);
       req.destroy();
       reject(new NetworkError("Request timed out."));
     });
     req.on("error", (err) => {
-      reject(new NetworkError(`Request failed: ${err.message}`, void 0, { cause: err }));
+      clearTimeout(totalTimer);
+      if (abortController.signal.aborted) {
+        reject(new NetworkError("Request timed out (total deadline exceeded)."));
+      } else {
+        reject(new NetworkError(`Request failed: ${err.message}`, void 0, { cause: err }));
+      }
     });
     req.end();
   });
@@ -381,24 +421,47 @@ async function fetchWithBrowser(url, resolvedIP, timeout) {
     );
   }
   const parsed = new URL(url);
-  const hostResolverRule = `MAP ${parsed.hostname} ${resolvedIP}`;
+  const pinnedAddr = resolvedIP.includes(":") ? `[${resolvedIP}]` : resolvedIP;
+  const hostResolverRule = `MAP ${parsed.hostname} ${pinnedAddr}`;
+  const chromiumArgs = [
+    "--disable-features=WebSockets",
+    `--host-resolver-rules=${hostResolverRule}`,
+    // Avoid /dev/shm exhaustion in Docker containers (default 64MB)
+    "--disable-dev-shm-usage"
+  ];
+  if (process.platform === "linux" && process.getuid?.() === 0) {
+    chromiumArgs.push("--no-sandbox");
+  }
   const browser = await playwright.chromium.launch({
     headless: true,
-    args: ["--disable-websockets", `--host-resolver-rules=${hostResolverRule}`]
+    args: chromiumArgs
   });
   try {
     const context = await browser.newContext({
       userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
     });
     const page = await context.newPage();
+    const initialHostname = parsed.hostname;
     await context.route("**/*", async (route) => {
       const requestUrl = route.request().url();
       try {
-        const parsed2 = new URL(requestUrl);
-        if (parsed2.protocol === "http:" || parsed2.protocol === "https:") {
-          await validateUrl(requestUrl);
+        const reqParsed = new URL(requestUrl);
+        if (reqParsed.protocol !== "http:" && reqParsed.protocol !== "https:") {
+          await route.continue();
+          return;
+        }
+        const subResolvedIP = await validateUrl(requestUrl);
+        if (reqParsed.hostname === initialHostname) {
+          await route.continue();
+          return;
         }
-        await route.continue();
+        const response = await pinnedRequest(requestUrl, subResolvedIP, safTimeout);
+        const contentType = typeof response.headers["content-type"] === "string" ? response.headers["content-type"] : "application/octet-stream";
+        await route.fulfill({
+          status: response.status,
+          contentType,
+          body: response.bodyBuffer
+        });
       } catch {
         await route.abort("blockedbyclient");
       }
@@ -416,7 +479,10 @@ async function fetchWithBrowser(url, resolvedIP, timeout) {
     }
     return { html, finalUrl: page.url() };
   } finally {
-    await browser.close();
+    try {
+      await browser.close();
+    } catch {
+    }
   }
 }
 async function fetchRawText(url, timeout) {
@@ -580,7 +646,7 @@ function extractMdx(html, url) {
   return extractMintlifyMdx(html, url);
 }
 function processRawMdx(mdx, url) {
-  const { body, metadata } = parseFrontmatter(mdx);
+  const { body, metadata } = parseFrontmatter(mdx.replace(/\r\n/g, "\n"));
   let markdown = stripMdxComponents(body);
   markdown = resolveRelativeUrls(markdown, url);
   markdown = markdown.replace(/\n{3,}/g, "\n\n");
@@ -1497,7 +1563,8 @@ function getPackageVersion() {
 async function validateOutputPath(outputPath) {
   const resolved = resolve(outputPath);
   const cwd = process.cwd();
-  if (!resolved.startsWith(cwd + "/") && resolved !== cwd) {
+  const cwdPrefix = cwd.endsWith(sep) ? cwd : cwd + sep;
+  if (!resolved.startsWith(cwdPrefix) && resolved !== cwd) {
     throw new Error(
       `Output path "${outputPath}" resolves outside the current directory.
 Resolved to: ${resolved}
@@ -1507,17 +1574,15 @@ For safety, output files must be within the working directory.`
   let checkPath = resolved;
   while (checkPath !== cwd && checkPath !== dirname(checkPath)) {
     if (existsSync(checkPath)) {
-      const stat = lstatSync(checkPath);
-      if (stat.isSymbolicLink()) {
-        const realTarget = await realpath(checkPath);
-        const realCwd = await realpath(cwd);
-        if (!realTarget.startsWith(realCwd + "/") && realTarget !== realCwd) {
-          throw new Error(
-            `Output path "${outputPath}" follows a symlink outside the current directory.
-Symlink "${checkPath}" points to "${realTarget}".
-For safety, output files must not escape the working directory via symlinks.`
-          );
-        }
+      const realTarget = await realpath(checkPath);
+      const realCwd = await realpath(cwd);
+      const realCwdPrefix = realCwd.endsWith(sep) ? realCwd : realCwd + sep;
+      if (!realTarget.startsWith(realCwdPrefix) && realTarget !== realCwd) {
+        throw new Error(
+          `Output path "${outputPath}" resolves outside the current directory.
+"${checkPath}" resolves to "${realTarget}".
+For safety, output files must not escape the working directory via symlinks or junctions.`
+        );
       }
       break;
     }
@@ -1583,6 +1648,9 @@ Received ${signal}, shutting down\u2026
   };
   process.on("SIGINT", onSignal);
   process.on("SIGTERM", onSignal);
+  if (process.platform === "win32") {
+    process.on("SIGBREAK", onSignal);
+  }
   await program.parseAsync(process.argv);
 }
 

package/dist/index.js

@@ -38,10 +38,10 @@ function expandIPv6(ip) {
   } else {
     groups = ip.split(":");
   }
-  return groups.map((g) => g.toLowerCase()).join(":");
+  return groups.map((g) => g.toLowerCase().padStart(4, "0")).join(":");
 }
 function normalizeIP(hostname) {
-  const bare = hostname.replace(/^\[|\]$/g, "");
+  const bare = hostname.replace(/^\[|\]$/g, "").replace(/%.*$/, "");
   if (isIP(bare) === 4) return bare;
   if (isIP(bare) === 6) {
     const ffmpDotted = bare.match(/^::ffff:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/i);
@@ -53,7 +53,9 @@ function normalizeIP(hostname) {
       return `${high >> 8 & 255}.${high & 255}.${low >> 8 & 255}.${low & 255}`;
     }
     const fullFfmpHex = expandIPv6(bare);
-    const fullFfmpMatch = fullFfmpHex.match(/^0:0:0:0:0:ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/i);
+    const fullFfmpMatch = fullFfmpHex.match(
+      /^0000:0000:0000:0000:0000:ffff:([0-9a-f]{4}):([0-9a-f]{4})$/i
+    );
     if (fullFfmpMatch) {
       const high = parseInt(fullFfmpMatch[1], 16);
       const low = parseInt(fullFfmpMatch[2], 16);
@@ -110,20 +112,21 @@ function isPrivateIP(ip) {
   if (ip.startsWith("198.51.100.")) return true;
   if (ip.startsWith("203.0.113.")) return true;
   if (/^198\.(18|19)\./.test(ip)) return true;
-  if (/^24[0-9]\./.test(ip) || ip.startsWith("255.")) return true;
+  if (ip.includes(".") && !ip.includes(":")) {
+    const firstOctet = parseInt(ip.split(".")[0], 10);
+    if (firstOctet >= 224) return true;
+  }
   const expanded = ip.includes("::") ? expandIPv6(ip) : ip.toLowerCase();
-  if (ip === "::1" || ip === "::" || expanded === "0:0:0:0:0:0:0:1" || expanded === "0:0:0:0:0:0:0:0")
+  if (ip === "::1" || ip === "::" || expanded === "0000:0000:0000:0000:0000:0000:0000:0001" || expanded === "0000:0000:0000:0000:0000:0000:0000:0000")
     return true;
   if (expanded.startsWith("fc") || expanded.startsWith("fd")) return true;
-  if (expanded.startsWith("fe80:") || expanded.startsWith("fe8") || expanded.startsWith("fe9") || expanded.startsWith("fea") || expanded.startsWith("feb"))
-    return true;
-  if (expanded.startsWith("fec0:") || expanded.startsWith("fec") || expanded.startsWith("fed") || expanded.startsWith("fee") || expanded.startsWith("fef"))
-    return true;
+  if (/^fe[89ab][0-9a-f]:/.test(expanded)) return true;
+  if (/^fe[cdef][0-9a-f]:/.test(expanded)) return true;
   if (expanded.startsWith("ff")) return true;
-  if (expanded.startsWith("64:ff9b:")) return true;
+  if (expanded.startsWith("0064:ff9b:")) return true;
   if (expanded.startsWith("2002:")) return true;
-  if (expanded.startsWith("2001:db8:")) return true;
-  if (expanded.startsWith("2001:0:")) return true;
+  if (expanded.startsWith("2001:0db8:")) return true;
+  if (expanded.startsWith("2001:0000:")) return true;
   return false;
 }
 
@@ -173,6 +176,8 @@ var BLOCKED_HOSTNAME_SUFFIXES = [
   ".internal",
   ".local",
   ".localhost",
+  ".localdomain",
+  ".intranet",
   ".corp",
   ".home",
   ".lan"
@@ -193,13 +198,18 @@ async function resolveAndValidateHostname(hostname) {
     return normalizedIP;
   }
   try {
-    const { address } = await lookup(hostname);
-    if (isPrivateIP(address)) {
-      throw new SSRFError(
-        `Blocked request to "${hostname}" \u2014 resolves to private IP. Requests to internal networks are not allowed.`
-      );
+    const results = await lookup(hostname, { all: true });
+    if (results.length === 0) {
+      throw new NetworkError(`DNS lookup for "${hostname}" returned no results. Check the URL.`);
     }
-    return address;
+    for (const { address } of results) {
+      if (isPrivateIP(address)) {
+        throw new SSRFError(
+          `Blocked request to "${hostname}" \u2014 resolves to private IP. Requests to internal networks are not allowed.`
+        );
+      }
+    }
+    return results[0].address;
   } catch (err) {
     if (err instanceof SSRFError) throw err;
     if (err instanceof Error) {
@@ -224,7 +234,7 @@ async function validateUrl(url) {
       `Unsupported protocol "${parsed.protocol}". Only HTTP and HTTPS are supported.`
     );
   }
-  const hostname = parsed.hostname.toLowerCase();
+  const hostname = parsed.hostname.toLowerCase().replace(/\.$/, "");
   if (BLOCKED_HOSTNAMES.has(hostname)) {
     throw new SSRFError(
       `Blocked request to "${hostname}". Requests to internal hosts are not allowed.`
@@ -239,6 +249,20 @@ async function validateUrl(url) {
   }
   return resolveAndValidateHostname(hostname);
 }
+function extractCharset(headers) {
+  const raw = headers["content-type"];
+  const contentType = typeof raw === "string" ? raw : "";
+  const match = contentType.match(/charset\s*=\s*["']?([^"';,\s]+)["']?/i);
+  return match ? match[1].toLowerCase() : null;
+}
+function decodeBody(buffer, charset) {
+  const encoding = charset || "utf-8";
+  try {
+    return new TextDecoder(encoding).decode(buffer);
+  } catch {
+    return buffer.toString("utf-8");
+  }
+}
 var HTML_CONTENT_TYPES = [
   "text/html",
   "application/xhtml+xml",
@@ -273,11 +297,14 @@ function pinnedRequest(url, resolvedIP, timeout) {
     const parsed = new URL(url);
     const isHttps = parsed.protocol === "https:";
     const requestFn = isHttps ? httpsRequest : httpRequest;
+    const abortController = new AbortController();
+    const totalTimer = setTimeout(() => abortController.abort(), timeout);
     const req = requestFn(
       url,
       {
         method: "GET",
         timeout,
+        signal: abortController.signal,
         lookup: createPinnedLookup(resolvedIP),
         headers: {
           "User-Agent": "Mozilla/5.0 (compatible; web-to-markdown/0.1; +https://github.com/nidhi-singh02/mark-it-down)",
@@ -291,6 +318,7 @@ function pinnedRequest(url, resolvedIP, timeout) {
         res.on("data", (chunk) => {
           totalBytes += chunk.length;
           if (totalBytes > MAX_RESPONSE_SIZE) {
+            clearTimeout(totalTimer);
             req.destroy();
             reject(
               new ContentError(
@@ -302,24 +330,36 @@ function pinnedRequest(url, resolvedIP, timeout) {
           chunks.push(chunk);
         });
         res.on("end", () => {
+          clearTimeout(totalTimer);
           const bodyBuffer = Buffer.concat(chunks);
+          const responseHeaders = res.headers;
+          const charset = extractCharset(responseHeaders);
           resolve({
             status: res.statusCode || 0,
-            headers: res.headers,
-            body: bodyBuffer.toString("utf-8"),
+            headers: responseHeaders,
+            body: decodeBody(bodyBuffer, charset),
             bodyBuffer,
             responseUrl: url
           });
         });
-        res.on("error", reject);
+        res.on("error", (err) => {
+          clearTimeout(totalTimer);
+          reject(err);
+        });
       }
     );
     req.on("timeout", () => {
+      clearTimeout(totalTimer);
       req.destroy();
       reject(new NetworkError("Request timed out."));
     });
     req.on("error", (err) => {
-      reject(new NetworkError(`Request failed: ${err.message}`, void 0, { cause: err }));
+      clearTimeout(totalTimer);
+      if (abortController.signal.aborted) {
+        reject(new NetworkError("Request timed out (total deadline exceeded)."));
+      } else {
+        reject(new NetworkError(`Request failed: ${err.message}`, void 0, { cause: err }));
+      }
     });
     req.end();
   });
@@ -372,24 +412,47 @@ async function fetchWithBrowser(url, resolvedIP, timeout) {
     );
   }
   const parsed = new URL(url);
-  const hostResolverRule = `MAP ${parsed.hostname} ${resolvedIP}`;
+  const pinnedAddr = resolvedIP.includes(":") ? `[${resolvedIP}]` : resolvedIP;
+  const hostResolverRule = `MAP ${parsed.hostname} ${pinnedAddr}`;
+  const chromiumArgs = [
+    "--disable-features=WebSockets",
+    `--host-resolver-rules=${hostResolverRule}`,
+    // Avoid /dev/shm exhaustion in Docker containers (default 64MB)
+    "--disable-dev-shm-usage"
+  ];
+  if (process.platform === "linux" && process.getuid?.() === 0) {
+    chromiumArgs.push("--no-sandbox");
+  }
   const browser = await playwright.chromium.launch({
     headless: true,
-    args: ["--disable-websockets", `--host-resolver-rules=${hostResolverRule}`]
+    args: chromiumArgs
   });
   try {
     const context = await browser.newContext({
       userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
     });
     const page = await context.newPage();
+    const initialHostname = parsed.hostname;
     await context.route("**/*", async (route) => {
       const requestUrl = route.request().url();
       try {
-        const parsed2 = new URL(requestUrl);
-        if (parsed2.protocol === "http:" || parsed2.protocol === "https:") {
-          await validateUrl(requestUrl);
+        const reqParsed = new URL(requestUrl);
+        if (reqParsed.protocol !== "http:" && reqParsed.protocol !== "https:") {
+          await route.continue();
+          return;
+        }
+        const subResolvedIP = await validateUrl(requestUrl);
+        if (reqParsed.hostname === initialHostname) {
+          await route.continue();
+          return;
         }
-        await route.continue();
+        const response = await pinnedRequest(requestUrl, subResolvedIP, safTimeout);
+        const contentType = typeof response.headers["content-type"] === "string" ? response.headers["content-type"] : "application/octet-stream";
+        await route.fulfill({
+          status: response.status,
+          contentType,
+          body: response.bodyBuffer
+        });
       } catch {
         await route.abort("blockedbyclient");
       }
@@ -407,7 +470,10 @@ async function fetchWithBrowser(url, resolvedIP, timeout) {
     }
     return { html, finalUrl: page.url() };
   } finally {
-    await browser.close();
+    try {
+      await browser.close();
+    } catch {
+    }
   }
 }
 async function fetchRawText(url, timeout) {
@@ -571,7 +637,7 @@ function extractMdx(html, url) {
   return extractMintlifyMdx(html, url);
 }
 function processRawMdx(mdx, url) {
-  const { body, metadata } = parseFrontmatter(mdx);
+  const { body, metadata } = parseFrontmatter(mdx.replace(/\r\n/g, "\n"));
   let markdown = stripMdxComponents(body);
   markdown = resolveRelativeUrls(markdown, url);
   markdown = markdown.replace(/\n{3,}/g, "\n\n");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web-to-markdown",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "description": "Convert any web page to clean Markdown. Built for developers and LLM pipelines.",
   "author": "Nidhi Singh",
   "repository": {
@@ -66,6 +66,7 @@
     "@types/node": "^22.0.0",
     "@types/turndown": "^5.0.5",
     "eslint": "^10.0.2",
+    "js-tiktoken": "^1.0.21",
     "prettier": "^3.8.1",
     "tsup": "^8.5.0",
     "typescript": "^5.7.0",