web-to-markdown-crawler 1.0.0 → 1.0.2

package/README.md

@@ -1,5 +1,8 @@
 # web-to-markdown-crawler
 
+[![npm version](https://img.shields.io/npm/v/web-to-markdown-crawler)](https://www.npmjs.com/package/web-to-markdown-crawler)
+[![CI](https://github.com/leochilds/web-to-markdown-crawler/actions/workflows/ci.yml/badge.svg)](https://github.com/leochilds/web-to-markdown-crawler/actions/workflows/ci.yml)
+
 A CLI tool that crawls a website and converts every page to Markdown, mirroring the site's URL structure as a local directory tree. Internal links are rewritten to relative `.md` paths so the output works as a self-contained document collection.
 
 ## Features

package/dist/index.js

@@ -68287,7 +68287,7 @@ var STRINGS = {
   COMMENT_END: toUint8Array("-->")
 };
 // node_modules/undici/index.js
-var __filename = "/home/leo/code/web-to-markdown-crawler/node_modules/undici/index.js";
+var __filename = "/home/runner/work/web-to-markdown-crawler/web-to-markdown-crawler/node_modules/undici/index.js";
 var Client = require_client();
 var Dispatcher = require_dispatcher();
 var Pool = require_pool();
@@ -68565,7 +68565,7 @@ function serializeGraph(graph, meta, startUrl, totalPages, crawledAt) {
   }
   return { startUrl, crawledAt, totalPages, nodes };
 }
-async function processPage(item, config, startHostname, visited, graph, meta, newItems, state) {
+async function processPage(item, config, visited, graph, meta, newItems, state) {
   const { url, depth } = item;
   if (config.maxPages !== undefined && state.totalPages >= config.maxPages) {
     const node2 = meta.get(url);
@@ -68584,11 +68584,16 @@ async function processPage(item, config, startHostname, visited, graph, meta, ne
         meta.set(effectiveUrl, { depth, status: "pending" });
       }
     }
+    if (depth === 0) {
+      const effectiveHostname = new URL(effectiveUrl).hostname;
+      state.startHostname = effectiveHostname;
+      state.internalHostnames.add(effectiveHostname);
+    }
     const links = extractLinks(html3, effectiveUrl);
     for (const rawLink of links) {
       const link = normalizeUrl2(rawLink);
       const linkHostname = new URL(link).hostname;
-      const isInternal = linkHostname === startHostname;
+      const isInternal = state.internalHostnames.has(linkHostname);
       graph.dir(url, link);
       if (!meta.has(link)) {
         meta.set(link, {
@@ -68604,7 +68609,7 @@ async function processPage(item, config, startHostname, visited, graph, meta, ne
     const outputPath = urlToOutputPath(effectiveUrl, config.outputDir);
     const relativeOutputPath = path3.relative(path3.resolve(config.outputDir), outputPath);
     const markdown = convertToMarkdown(html3, effectiveUrl);
-    const rewritten = rewriteInternalLinks(markdown, startHostname, relativeOutputPath, effectiveUrl);
+    const rewritten = rewriteInternalLinks(markdown, state.startHostname, relativeOutputPath, effectiveUrl);
     await writePage(outputPath, rewritten);
     const node2 = meta.get(url);
     node2.status = "success";
@@ -68620,12 +68625,16 @@ async function processPage(item, config, startHostname, visited, graph, meta, ne
 }
 async function run(config) {
   const startUrl = normalizeUrl2(config.startUrl);
-  const startHostname = new URL(startUrl).hostname;
   const crawledAt = new Date().toISOString();
   const visited = new Set([startUrl]);
   const graph = createGraph();
   const meta = new Map;
-  const state = { totalPages: 0 };
+  const initialHostname = new URL(startUrl).hostname;
+  const state = {
+    totalPages: 0,
+    startHostname: initialHostname,
+    internalHostnames: new Set([initialHostname])
+  };
   meta.set(startUrl, { depth: 0, status: "pending" });
   const limit = pLimit(config.concurrency);
   let queue = [{ url: startUrl, depth: 0 }];
@@ -68634,7 +68643,7 @@ async function run(config) {
       break;
     const batch = queue.splice(0);
     const newItems = [];
-    const tasks = batch.map((item) => limit(() => processPage(item, config, startHostname, visited, graph, meta, newItems, state)));
+    const tasks = batch.map((item) => limit(() => processPage(item, config, visited, graph, meta, newItems, state)));
     await Promise.allSettled(tasks);
     queue = newItems;
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web-to-markdown-crawler",
-  "version": "1.0.0",
+  "version": "1.0.2",
   "description": "Web crawler that converts site pages to markdown, mirroring the URL structure locally",
   "type": "module",
   "main": "dist/index.js",
@@ -10,6 +10,10 @@
   "files": [
     "dist"
   ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/leochilds/web-to-markdown-crawler.git"
+  },
   "engines": {
     "node": ">=18"
   },