web-to-markdown-crawler 1.0.2 → 1.0.3

package/dist/index.js

@@ -68592,16 +68592,17 @@ async function processPage(item, config, visited, graph, meta, newItems, state)
     const links = extractLinks(html3, effectiveUrl);
     for (const rawLink of links) {
       const link = normalizeUrl2(rawLink);
-      const linkHostname = new URL(link).hostname;
-      const isInternal = state.internalHostnames.has(linkHostname);
+      const parsedLink = new URL(link);
+      const isInternal = state.internalHostnames.has(parsedLink.hostname);
+      const isExcluded = config.exclude?.some((pattern) => parsedLink.pathname.startsWith(pattern)) ?? false;
       graph.dir(url, link);
       if (!meta.has(link)) {
         meta.set(link, {
           depth: depth + 1,
-          status: isInternal ? "pending" : "skipped"
+          status: isInternal && !isExcluded ? "pending" : "skipped"
         });
       }
-      if (isInternal && !visited.has(link) && (config.maxDepth === undefined || depth + 1 <= config.maxDepth)) {
+      if (isInternal && !isExcluded && !visited.has(link) && (config.maxDepth === undefined || depth + 1 <= config.maxDepth)) {
         visited.add(link);
         newItems.push({ url: link, depth: depth + 1 });
       }
@@ -68669,7 +68670,7 @@ function parseStartUrl(raw) {
   }
 }
 var program2 = new Command;
-program2.name("crawl").description("Crawl a website and convert pages to markdown").argument("<url>", "Starting URL to crawl").option("-o, --output <dir>", "Output directory", "./output").option("-c, --concurrency <n>", "Number of parallel fetches", "5").option("--max-depth <n>", "Maximum crawl depth (0 = start page only)").option("--max-pages <n>", "Maximum number of pages to crawl").option("--delay <ms>", "Delay in milliseconds between requests").action(async (url, opts) => {
+program2.name("crawl").description("Crawl a website and convert pages to markdown").argument("<url>", "Starting URL to crawl").option("-o, --output <dir>", "Output directory", "./output").option("-c, --concurrency <n>", "Number of parallel fetches", "5").option("--max-depth <n>", "Maximum crawl depth (0 = start page only)").option("--max-pages <n>", "Maximum number of pages to crawl").option("--delay <ms>", "Delay in milliseconds between requests").option("--exclude <path>", "Exclude a URL path prefix from crawling — can be repeated (e.g. --exclude /admin --exclude /login)", (val2, prev2) => [...prev2, val2], []).action(async (url, opts) => {
   const startUrl = parseStartUrl(url);
   const concurrency = parseInt(opts["concurrency"] ?? "5", 10);
   if (isNaN(concurrency) || concurrency < 1) {
@@ -68679,6 +68680,8 @@ program2.name("crawl").description("Crawl a website and convert pages to markdow
   const maxDepth = opts["maxDepth"] != null ? parseInt(opts["maxDepth"], 10) : undefined;
   const maxPages = opts["maxPages"] != null ? parseInt(opts["maxPages"], 10) : undefined;
   const delayMs = opts["delay"] != null ? parseInt(opts["delay"], 10) : undefined;
+  const excludeRaw = opts["exclude"];
+  const exclude = excludeRaw.length ? excludeRaw : undefined;
   if (maxDepth !== undefined && isNaN(maxDepth)) {
     console.error("Error: --max-depth must be an integer");
     process.exit(1);
@@ -68697,7 +68700,8 @@ program2.name("crawl").description("Crawl a website and convert pages to markdow
     concurrency,
     maxDepth,
     maxPages,
-    delayMs
+    delayMs,
+    exclude
   };
   console.log(`Starting crawl of ${startUrl}`);
   console.log(`Output: ${config.outputDir} | Concurrency: ${config.concurrency}${maxDepth !== undefined ? ` | Max depth: ${maxDepth}` : ""}${maxPages !== undefined ? ` | Max pages: ${maxPages}` : ""}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web-to-markdown-crawler",
-  "version": "1.0.2",
+  "version": "1.0.3",
   "description": "Web crawler that converts site pages to markdown, mirroring the URL structure locally",
   "type": "module",
   "main": "dist/index.js",