site-mirror 1.0.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Mahesh
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,101 @@
+# site-mirror
+
+A CLI tool to mirror websites for offline browsing using Playwright.
+
+## Installation
+
+```bash
+# Install globally
+npm install -g site-mirror
+
+# Or use directly via npx
+npx site-mirror --help
+```
+
+## Quick Start
+
+```bash
+# Download a single page with all its assets (no config needed!)
+site-mirror run --start https://www.apple.com/iphone/ --singlePage
+
+# Crawl an entire site
+site-mirror run --start https://example.com/
+
+# Or use interactive config-based workflow:
+site-mirror init          # Interactive prompts to create site-mirror.config.json
+site-mirror run           # Runs the mirror using config
+site-mirror serve         # Serve locally on port 8080
+```
+
+## Commands
+
+| Command                  | Description                                           |
+| ------------------------ | ----------------------------------------------------- |
+| `site-mirror init`       | Interactive setup - creates `site-mirror.config.json` |
+| `site-mirror run`        | Run the mirror (reads config + CLI overrides)         |
+| `site-mirror serve`      | Serve the `./offline` folder locally                  |
+| `site-mirror serve 3000` | Serve on a custom port                                |
+
+## CLI Options (for `run`)
+
+| Option              | Description                              | Default         |
+| ------------------- | ---------------------------------------- | --------------- |
+| `--start <url>`     | Start URL (required if not in config)    | -               |
+| `--out <dir>`       | Output directory                         | `./offline`     |
+| `--maxPages <n>`    | Max pages to crawl (0 = unlimited)       | `0`             |
+| `--maxDepth <n>`    | Max link depth (0 = unlimited)           | `0`             |
+| `--sameOriginOnly`  | Only crawl same-origin pages             | `true`          |
+| `--seedSitemaps`    | Seed URLs from sitemap.xml/robots.txt    | `false`         |
+| `--singlePage`      | Download only this page + all its assets | `false`         |
+
+## Config File (`site-mirror.config.json`)
+
+Created via `site-mirror init` (interactive) or manually:
+
+```json
+{
+  "start": "https://example.com/",
+  "out": "./offline",
+  "singlePage": false,
+  "maxPages": 200,
+  "maxDepth": 6,
+  "sameOriginOnly": true,
+  "seedSitemaps": false
+}
+```
+
+CLI options override config file settings.
+
+## Output Structure
+
+```
+./offline/
+├── index.html              # Homepage
+├── about/
+│   └── index.html          # /about/ page
+├── _next/                   # Same-origin assets
+│   └── static/
+├── _external/               # Cross-origin assets
+│   └── cdn.example.com/
+│       └── script.js
+```
+
+## How It Works
+
+1. Launches headless Chromium via Playwright
+2. Navigates to each page, waits for network idle
+3. Captures all static assets (CSS, JS, images, fonts, videos)
+4. Rewrites absolute same-origin URLs to relative paths
+5. Injects a script to handle SPA-style navigation offline
+6. Discovers new pages via `<a href>` links
+7. Saves everything to the output directory
+
+## Notes
+
+- XHR/fetch API responses are not saved (only rendered HTML + static assets)
+- Some interactive features requiring live APIs won't work offline
+- Be mindful of target site's Terms of Service and robots.txt
+
+## License
+
+MIT

package/bin/cli.mjs

@@ -0,0 +1,286 @@
+#!/usr/bin/env node
+/**
+ * site-mirror CLI
+ *
+ * Usage:
+ *   site-mirror init                  Interactive setup - creates site-mirror.config.json
+ *   site-mirror run [options]         Run the mirror (reads config + CLI overrides)
+ *   site-mirror serve [port]          Serve the offline folder locally
+ *
+ * Options (for run):
+ *   --start <url>         Start URL (required if not in config)
+ *   --out <dir>           Output directory (default: ./offline)
+ *   --maxPages <n>        Max pages to crawl (0 = unlimited)
+ *   --maxDepth <n>        Max link depth (0 = unlimited)
+ *   --sameOriginOnly      Only crawl same-origin pages (default: true)
+ *   --seedSitemaps        Seed URLs from sitemap.xml (default: false)
+ *   --singlePage          Download only this one page + all its assets (no crawling)
+ */
+
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import readline from 'node:readline';
+import { fileURLToPath } from 'node:url';
+import { execSync, spawn } from 'node:child_process';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const CONFIG_FILE = 'site-mirror.config.json';
+
+const defaultConfig = {
+  start: '',
+  out: './offline',
+  maxPages: 0,
+  maxDepth: 0,
+  sameOriginOnly: true,
+  seedSitemaps: false,
+  singlePage: false,
+  userAgent:
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36',
+};
+
+async function loadConfig(cwd) {
+  const configPath = path.join(cwd, CONFIG_FILE);
+  try {
+    const raw = await fs.readFile(configPath, 'utf-8');
+    return { ...defaultConfig, ...JSON.parse(raw) };
+  } catch {
+    return { ...defaultConfig };
+  }
+}
+
+function parseCliArgs(argv) {
+  const args = {};
+  for (let i = 0; i < argv.length; i++) {
+    const key = argv[i];
+    const next = argv[i + 1];
+
+    if (key === '--start') {
+      args.start = next;
+      i++;
+    } else if (key === '--out') {
+      args.out = next;
+      i++;
+    } else if (key === '--maxPages') {
+      args.maxPages = Number(next);
+      i++;
+    } else if (key === '--maxDepth') {
+      args.maxDepth = Number(next);
+      i++;
+    } else if (key === '--sameOriginOnly') {
+      args.sameOriginOnly = next !== 'false' && next !== '0';
+      i++;
+    } else if (key === '--seedSitemaps') {
+      args.seedSitemaps = next === 'true' || next === '1' || next === 'yes';
+      i++;
+    } else if (key === '--singlePage') {
+      args.singlePage = true;
+    } else if (key === '--userAgent') {
+      args.userAgent = next;
+      i++;
+    }
+  }
+  return args;
+}
+
+async function prompt(rl, question, defaultValue) {
+  return new Promise((resolve) => {
+    const hint = defaultValue !== undefined ? ` (${defaultValue})` : '';
+    rl.question(`${question}${hint}: `, (answer) => {
+      const trimmed = answer.trim();
+      resolve(trimmed === '' && defaultValue !== undefined ? defaultValue : trimmed);
+    });
+  });
+}
+
+async function cmdInit(cwd) {
+  const configPath = path.join(cwd, CONFIG_FILE);
+  const exists = await fs
+    .access(configPath)
+    .then(() => true)
+    .catch(() => false);
+
+  if (exists) {
+    console.log(`Config already exists: ${configPath}`);
+    console.log('Delete it first if you want to reinitialize.');
+    return;
+  }
+
+  console.log('');
+  console.log('Welcome to site-mirror! Let\'s set up your config.');
+  console.log('Press Enter to accept the default value shown in parentheses.');
+  console.log('');
+
+  const rl = readline.createInterface({
+    input: process.stdin,
+    output: process.stdout,
+  });
+
+  try {
+    const start = await prompt(rl, 'Website URL to mirror', 'https://example.com/');
+    const out = await prompt(rl, 'Output directory', './offline');
+    const singlePageStr = await prompt(rl, 'Single page only? (yes/no)', 'no');
+    const singlePage = singlePageStr === 'yes' || singlePageStr === 'y' || singlePageStr === 'true';
+
+    let maxPages = 0;
+    let maxDepth = 0;
+    let seedSitemaps = false;
+
+    if (!singlePage) {
+      const maxPagesStr = await prompt(rl, 'Max pages to crawl (0 = unlimited)', '200');
+      maxPages = parseInt(maxPagesStr, 10) || 0;
+
+      const maxDepthStr = await prompt(rl, 'Max link depth (0 = unlimited)', '6');
+      maxDepth = parseInt(maxDepthStr, 10) || 0;
+
+      const seedStr = await prompt(rl, 'Seed from sitemap.xml? (yes/no)', 'no');
+      seedSitemaps = seedStr === 'yes' || seedStr === 'y' || seedStr === 'true';
+    }
+
+    const config = {
+      start,
+      out,
+      singlePage,
+      maxPages,
+      maxDepth,
+      sameOriginOnly: true,
+      seedSitemaps,
+    };
+
+    await fs.writeFile(configPath, JSON.stringify(config, null, 2) + '\n');
+    console.log('');
+    console.log(`Created ${CONFIG_FILE}`);
+    console.log('');
+    console.log('Run "site-mirror run" to start mirroring!');
+  } finally {
+    rl.close();
+  }
+}
+
+async function cmdRun(cwd, cliArgs) {
+  const config = await loadConfig(cwd);
+  const merged = { ...config, ...cliArgs };
+
+  // Validate required fields
+  if (!merged.start) {
+    console.error('');
+    console.error('Error: No start URL specified.');
+    console.error('');
+    console.error('You can either:');
+    console.error('  1. Run "site-mirror init" to create a config file');
+    console.error('  2. Pass --start <url> directly, e.g.:');
+    console.error('     site-mirror run --start https://example.com/');
+    console.error('     site-mirror run --start https://example.com/page --singlePage');
+    console.error('');
+    process.exit(1);
+  }
+
+  // Validate URL format
+  try {
+    new URL(merged.start);
+  } catch {
+    console.error('');
+    console.error(`Error: Invalid URL "${merged.start}"`);
+    console.error('Please provide a valid URL starting with http:// or https://');
+    console.error('');
+    process.exit(1);
+  }
+
+  console.log('');
+  console.log('Configuration:');
+  console.log(`  start:          ${merged.start}`);
+  console.log(`  out:            ${merged.out}`);
+  if (merged.singlePage) {
+    console.log(`  mode:           SINGLE PAGE (no crawling)`);
+  } else {
+    console.log(`  maxPages:       ${merged.maxPages || 'unlimited'}`);
+    console.log(`  maxDepth:       ${merged.maxDepth || 'unlimited'}`);
+    console.log(`  seedSitemaps:   ${merged.seedSitemaps}`);
+  }
+  console.log(`  sameOriginOnly: ${merged.sameOriginOnly}`);
+  console.log('');
+
+  // Dynamically import the mirror lib
+  const { mirrorSite } = await import('../lib/mirror.mjs');
+  await mirrorSite({
+    start: merged.start,
+    out: path.resolve(cwd, merged.out),
+    maxPages: merged.singlePage ? 1 : merged.maxPages,
+    maxDepth: merged.singlePage ? 0 : merged.maxDepth,
+    sameOriginOnly: merged.sameOriginOnly,
+    seedSitemaps: merged.singlePage ? false : merged.seedSitemaps,
+    singlePage: merged.singlePage,
+    userAgent: merged.userAgent,
+  });
+}
+
+async function cmdServe(cwd, port) {
+  const outDir = path.join(cwd, 'offline');
+  const exists = await fs
+    .access(outDir)
+    .then(() => true)
+    .catch(() => false);
+
+  if (!exists) {
+    console.error(`No offline folder found at ${outDir}`);
+    console.error('Run "site-mirror run" first to download the site.');
+    process.exit(1);
+  }
+
+  console.log(`Serving ${outDir} on http://localhost:${port}/`);
+  try {
+    execSync(`npx http-server "${outDir}" -p ${port} -c-1`, { stdio: 'inherit' });
+  } catch {
+    // User closed with Ctrl+C
+  }
+}
+
+async function main() {
+  const cwd = process.cwd();
+  const args = process.argv.slice(2);
+  const command = args[0];
+
+  if (!command || command === 'help' || command === '--help' || command === '-h') {
+    console.log(`
+site-mirror - Mirror websites for offline browsing
+
+Usage:
+  site-mirror init               Interactive setup - creates site-mirror.config.json
+  site-mirror run [options]      Mirror the website (reads config + CLI overrides)
+  site-mirror serve [port]       Serve the offline folder (default port: 8080)
+
+Run options:
+  --start <url>         Start URL (required if not in config)
+  --out <dir>           Output directory (default: ./offline)
+  --maxPages <n>        Max pages (0 = unlimited)
+  --maxDepth <n>        Max depth (0 = unlimited)
+  --seedSitemaps true   Seed from sitemap.xml
+  --singlePage          Download only this page + all its assets (no crawling)
+
+Examples:
+  site-mirror run --start https://example.com/page --singlePage
+  site-mirror init
+  site-mirror run
+  site-mirror serve 3000
+`);
+    return;
+  }
+
+  if (command === 'init') {
+    await cmdInit(cwd);
+  } else if (command === 'run') {
+    const cliArgs = parseCliArgs(args.slice(1));
+    await cmdRun(cwd, cliArgs);
+  } else if (command === 'serve') {
+    const port = args[1] || '8080';
+    await cmdServe(cwd, port);
+  } else {
+    // Assume it's run with options directly (no subcommand)
+    const cliArgs = parseCliArgs(args);
+    await cmdRun(cwd, cliArgs);
+  }
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});

package/bin/postinstall.mjs

@@ -0,0 +1,15 @@
+#!/usr/bin/env node
+/**
+ * Post-install script to ensure Playwright browsers are available.
+ * Runs automatically after `npm install`.
+ */
+import { execSync } from 'node:child_process';
+
+console.log('[site-mirror] Checking Playwright browsers...');
+
+try {
+  execSync('npx playwright install chromium', { stdio: 'inherit' });
+  console.log('[site-mirror] Chromium browser ready.');
+} catch (err) {
+  console.warn('[site-mirror] Could not auto-install Chromium. Run manually: npx playwright install chromium');
+}

package/lib/mirror.mjs

@@ -0,0 +1,463 @@
+/**
+ * Core mirroring logic.
+ * Exports mirrorSite() for use by the CLI.
+ */
+import { chromium } from 'playwright';
+import * as cheerio from 'cheerio';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import crypto from 'node:crypto';
+
+async function ensureDir(dirPath) {
+  await fs.mkdir(dirPath, { recursive: true });
+}
+
+function normalizePageUrl(urlString) {
+  const u = new URL(urlString);
+  u.hash = '';
+  return u;
+}
+
+function shouldSkipLink(url) {
+  const ext = path.posix.extname(url.pathname).toLowerCase();
+  const skipExt = new Set([
+    '.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg',
+    '.css', '.js', '.mjs', '.map',
+    '.json', '.xml', '.txt',
+    '.woff', '.woff2', '.ttf', '.otf',
+    '.pdf', '.zip', '.rar', '.7z',
+    '.mp4', '.webm', '.mp3', '.wav',
+  ]);
+  return skipExt.has(ext);
+}
+
+function pageOutputPath(outRoot, pageUrl) {
+  const pathname = decodeURIComponent(pageUrl.pathname || '/');
+  let rel;
+
+  if (pathname === '/' || pathname === '') {
+    rel = 'index.html';
+  } else if (pathname.endsWith('/')) {
+    rel = path.posix.join(pathname, 'index.html').slice(1);
+  } else {
+    const ext = path.posix.extname(pathname);
+    if (ext) rel = pathname.slice(1);
+    else rel = path.posix.join(pathname, 'index.html').slice(1);
+  }
+
+  if (pageUrl.search) {
+    const hash = crypto.createHash('sha1').update(pageUrl.search).digest('hex').slice(0, 8);
+    const dir = path.posix.dirname(rel);
+    const base = path.posix.basename(rel, '.html');
+    rel = path.posix.join(dir === '.' ? '' : dir, `${base}__qs-${hash}.html`);
+  }
+
+  return path.join(outRoot, rel);
+}
+
+function guessExtension(contentType) {
+  const ct = (contentType || '').toLowerCase();
+  if (ct.includes('text/css')) return '.css';
+  if (ct.includes('javascript')) return '.js';
+  if (ct.includes('application/wasm')) return '.wasm';
+  if (ct.includes('image/svg+xml')) return '.svg';
+  if (ct.startsWith('image/')) return `.${ct.split('image/')[1].split(';')[0]}`;
+  if (ct.includes('font/woff2')) return '.woff2';
+  if (ct.includes('font/woff')) return '.woff';
+  if (ct.includes('font/ttf')) return '.ttf';
+  if (ct.includes('font/otf')) return '.otf';
+  return '';
+}
+
+function assetOutputPath(outRoot, startOrigin, assetUrl, contentType) {
+  const pathname = assetUrl.pathname || '/';
+  const ext = path.posix.extname(pathname);
+
+  if (assetUrl.origin === startOrigin) {
+    const rel = pathname === '/' ? 'index' : pathname.slice(1);
+    return path.join(outRoot, rel);
+  }
+
+  const hostRoot = path.posix.join('_external', assetUrl.hostname);
+  let rel = pathname === '/' ? 'index' : pathname.slice(1);
+  rel = path.posix.join(hostRoot, rel);
+
+  if (assetUrl.search) {
+    const hash = crypto.createHash('sha1').update(assetUrl.search).digest('hex').slice(0, 8);
+    const dir = path.posix.dirname(rel);
+    const base = path.posix.basename(rel, ext || '');
+    const finalExt = ext || guessExtension(contentType) || '';
+    rel = path.posix.join(dir === '.' ? '' : dir, `${base}__qs-${hash}${finalExt}`);
+  } else if (!ext) {
+    const guessed = guessExtension(contentType);
+    if (guessed) rel = `${rel}${guessed}`;
+  }
+
+  return path.join(outRoot, rel);
+}
+
+function rewriteAbsoluteSameOriginUrls(html, startOrigin) {
+  const $ = cheerio.load(html);
+  const attrs = [
+    { selector: 'a', attr: 'href' },
+    { selector: 'link', attr: 'href' },
+    { selector: 'script', attr: 'src' },
+    { selector: 'img', attr: 'src' },
+    { selector: 'source', attr: 'src' },
+    { selector: 'source', attr: 'srcset' },
+    { selector: 'img', attr: 'srcset' },
+    { selector: 'video', attr: 'src' },
+    { selector: 'video', attr: 'poster' },
+    { selector: 'audio', attr: 'src' },
+    { selector: 'form', attr: 'action' },
+  ];
+
+  for (const { selector, attr } of attrs) {
+    $(selector).each((_, el) => {
+      const value = $(el).attr(attr);
+      if (!value) return;
+      if (value.startsWith('mailto:') || value.startsWith('tel:') || value.startsWith('javascript:')) return;
+
+      // Handle srcset specially (comma-separated list of URLs)
+      if (attr === 'srcset') {
+        const parts = value.split(',').map((part) => {
+          const trimmed = part.trim();
+          const [url, descriptor] = trimmed.split(/\s+/);
+          try {
+            const normalized = url.startsWith('//') ? `https:${url}` : url;
+            const u = new URL(normalized, startOrigin);
+            if (u.origin === startOrigin) {
+              return descriptor ? `${u.pathname}${u.search} ${descriptor}` : u.pathname + u.search;
+            }
+          } catch {}
+          return trimmed;
+        });
+        $(el).attr(attr, parts.join(', '));
+        return;
+      }
+
+      try {
+        const normalized = value.startsWith('//') ? `https:${value}` : value;
+        const u = new URL(normalized, startOrigin);
+        if (u.origin === startOrigin) {
+          $(el).attr(attr, `${u.pathname}${u.search}${u.hash}`);
+        }
+      } catch {
+        // ignore
+      }
+    });
+  }
+
+  return $.html();
+}
+
+function injectOfflineFullNavigation(html) {
+  const $ = cheerio.load(html);
+  const script = `\n<script>(function(){function a(e){for(;e&&e!==document.body;){if(e.tagName==='A')return e;e=e.parentElement}return null}document.addEventListener('click',function(e){if(e.defaultPrevented)return;if(e.button!==0)return;if(e.metaKey||e.ctrlKey||e.shiftKey||e.altKey)return;var t=a(e.target);if(!t)return;var h=t.getAttribute('href');if(!h||h.startsWith('#'))return;if(h.startsWith('http://')||h.startsWith('https://')||h.startsWith('mailto:')||h.startsWith('tel:'))return;var tg=t.getAttribute('target');if(tg&&tg!=='_self')return;e.preventDefault();window.location.href=h;},true);})();</script>\n`;
+  if ($('body').length) $('body').append(script);
+  else $.root().append(script);
+  return $.html();
+}
+
+async function writeFileSafe(filePath, bufferOrString) {
+  await ensureDir(path.dirname(filePath));
+  await fs.writeFile(filePath, bufferOrString);
+}
+
+function parseSitemapLocs(xmlText) {
+  const locs = [];
+  const re = /<loc>([^<]+)<\/loc>/gi;
+  let m;
+  while ((m = re.exec(xmlText)) !== null) {
+    const loc = m[1].trim();
+    if (loc) locs.push(loc);
+  }
+  return locs;
+}
+
+async function fetchText(url) {
+  const controller = new AbortController();
+  const timeoutMs = 20000;
+  const timeout = setTimeout(() => controller.abort(), timeoutMs);
+
+  try {
+    const res = await fetch(url, {
+      redirect: 'follow',
+      headers: { 'user-agent': 'offline-mirror/1.0' },
+      signal: controller.signal,
+    });
+    if (!res.ok) throw new Error(`HTTP ${res.status} for ${url}`);
+    return await res.text();
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+
+async function discoverSitemapUrls(startOrigin) {
+  const urls = new Set();
+
+  try {
+    const robots = await fetchText(`${startOrigin}/robots.txt`);
+    for (const line of robots.split(/\r?\n/)) {
+      const m = /^sitemap:\s*(.+)$/i.exec(line.trim());
+      if (m?.[1]) urls.add(m[1].trim());
+    }
+  } catch {
+    // ignore
+  }
+
+  urls.add(`${startOrigin}/sitemap.xml`);
+  return [...urls];
+}
+
+async function seedFromSitemaps({ startOrigin, enqueue }) {
+  const sitemapUrls = await discoverSitemapUrls(startOrigin);
+  const seenSitemaps = new Set();
+  const queue = [...sitemapUrls];
+
+  while (queue.length) {
+    const sitemapUrl = queue.shift();
+    if (seenSitemaps.has(sitemapUrl)) continue;
+    seenSitemaps.add(sitemapUrl);
+
+    console.log(`Sitemap: ${sitemapUrl}`);
+
+    let xml;
+    try {
+      xml = await fetchText(sitemapUrl);
+    } catch (e) {
+      console.log(`  (skipped: ${e?.message || 'fetch failed'})`);
+      continue;
+    }
+
+    let count = 0;
+    for (const loc of parseSitemapLocs(xml)) {
+      let u;
+      try {
+        u = new URL(loc);
+      } catch {
+        continue;
+      }
+
+      if (u.pathname.endsWith('.xml') && u.origin === startOrigin && /sitemap/i.test(u.pathname)) {
+        queue.push(u.toString());
+        continue;
+      }
+
+      if (u.origin !== startOrigin) continue;
+      u.hash = '';
+      enqueue(u.toString());
+      count++;
+    }
+    console.log(`  (found ${count} URLs)`);
+  }
+}
+
+export async function mirrorSite({ start, out, maxPages, maxDepth, sameOriginOnly, seedSitemaps, singlePage, userAgent }) {
+  const startUrl = normalizePageUrl(start);
+  const startOrigin = startUrl.origin;
+  const outRoot = path.resolve(out);
+
+  await ensureDir(outRoot);
+
+  const visitedPages = new Set();
+  const enqueuedPages = new Set();
+  const savedAssets = new Set();
+
+  const queue = [{ url: startUrl.toString(), depth: 0 }];
+  enqueuedPages.add(startUrl.toString());
+
+  const enqueue = (url, depth = 1) => {
+    if (enqueuedPages.has(url) || visitedPages.has(url)) return;
+    queue.push({ url, depth });
+    enqueuedPages.add(url);
+  };
+
+  if (seedSitemaps) {
+    console.log('Seeding from sitemap/robots...');
+    try {
+      await seedFromSitemaps({
+        startOrigin,
+        enqueue: (url) => {
+          try {
+            const u = normalizePageUrl(url);
+            if (sameOriginOnly && u.origin !== startOrigin) return;
+            if (shouldSkipLink(u)) return;
+            enqueue(u.toString(), 1);
+          } catch {
+            // ignore
+          }
+        },
+      });
+    } catch (e) {
+      console.warn(`Sitemap seeding failed: ${e?.message || e}`);
+    }
+    console.log(`Seeding complete. Queue size: ${queue.length}`);
+  }
+
+  let browser = await chromium.launch({ headless: true });
+  let context = await browser.newContext({ userAgent });
+
+  const relaunch = async (reason) => {
+    try {
+      console.warn(`Relaunching browser context: ${reason}`);
+      await context?.close();
+    } catch {}
+    try {
+      await browser?.close();
+    } catch {}
+    browser = await chromium.launch({ headless: true });
+    context = await browser.newContext({ userAgent });
+  };
+
+  const unlimitedPages = !maxPages || maxPages <= 0;
+  const unlimitedDepth = !maxDepth || maxDepth <= 0;
+
+  try {
+    while (queue.length > 0 && (unlimitedPages || visitedPages.size < maxPages)) {
+      const { url, depth } = queue.shift();
+      if (visitedPages.has(url)) continue;
+      if (!unlimitedDepth && depth > maxDepth) continue;
+
+      const pageUrl = normalizePageUrl(url);
+      if (sameOriginOnly && pageUrl.origin !== startOrigin) continue;
+
+      visitedPages.add(pageUrl.toString());
+      const progressMax = unlimitedPages ? '∞' : String(maxPages);
+      console.log(`[${visitedPages.size}/${progressMax}] ${pageUrl.toString()}`);
+
+      let page;
+      try {
+        page = await context.newPage();
+      } catch (e) {
+        await relaunch(e?.message || 'newPage failed');
+        page = await context.newPage();
+      }
+
+      const onResponse = async (response) => {
+        try {
+          const req = response.request();
+          if (req.method() !== 'GET') return;
+
+          const resourceType = req.resourceType();
+          if (['xhr', 'fetch', 'websocket', 'eventsource'].includes(resourceType)) return;
+
+          const responseUrl = response.url();
+          if (!responseUrl.startsWith('http://') && !responseUrl.startsWith('https://')) return;
+
+          const assetUrl = new URL(responseUrl);
+          const ct = response.headers()['content-type'] || '';
+
+          const isStaticLike = ['stylesheet', 'script', 'image', 'font', 'media', 'other'].includes(resourceType);
+          if (!isStaticLike) return;
+          if (ct.toLowerCase().includes('text/html') || resourceType === 'document') return;
+
+          const ctLower = ct.toLowerCase();
+          const allowed =
+            ctLower.includes('text/css') ||
+            ctLower.includes('javascript') ||
+            ctLower.startsWith('image/') ||
+            ctLower.startsWith('font/') ||
+            ctLower.startsWith('video/') ||
+            ctLower.startsWith('audio/') ||
+            ctLower.includes('font-woff') ||
+            ctLower.includes('application/wasm') ||
+            ctLower.includes('application/octet-stream');
+          if (!allowed) return;
+
+          const key = assetUrl.toString();
+          if (savedAssets.has(key)) return;
+
+          const body = await response.body();
+          const filePath = assetOutputPath(outRoot, startOrigin, assetUrl, ct);
+          await writeFileSafe(filePath, body);
+          savedAssets.add(key);
+        } catch {
+          // best effort
+        }
+      };
+
+      page.on('response', onResponse);
+
+      try {
+        try {
+          await page.goto(pageUrl.toString(), { waitUntil: 'domcontentloaded', timeout: 60000 });
+        } catch (e) {
+          const message = e?.message || String(e);
+          if (message.includes('net::ERR_ABORTED')) {
+            await page.goto(pageUrl.toString(), { waitUntil: 'load', timeout: 60000 });
+          } else {
+            throw e;
+          }
+        }
+
+        try {
+          await page.waitForLoadState('networkidle', { timeout: 20000 });
+        } catch {
+          // long-lived connections are fine
+        }
+
+        try {
+          await page.waitForTimeout(1500);
+        } catch {}
+
+        // Skip link discovery in single-page mode
+        if (!singlePage) {
+          const rawLinks = await page.$$eval('a[href]', (els) =>
+            els.map((e) => e.getAttribute('href')).filter(Boolean)
+          );
+
+          for (const href of rawLinks) {
+            if (!href || href.startsWith('#')) continue;
+            if (href.startsWith('mailto:') || href.startsWith('tel:') || href.startsWith('javascript:')) continue;
+
+            let linkUrl;
+            try {
+              linkUrl = new URL(href, pageUrl.toString());
+            } catch {
+              continue;
+            }
+
+            if (sameOriginOnly && linkUrl.origin !== startOrigin) continue;
+            linkUrl.hash = '';
+            if (shouldSkipLink(linkUrl)) continue;
+
+            const normalized = linkUrl.toString();
+            if (!enqueuedPages.has(normalized) && !visitedPages.has(normalized)) {
+              enqueue(normalized, depth + 1);
+            }
+          }
+        }
+
+        let html = await page.content();
+        html = rewriteAbsoluteSameOriginUrls(html, startOrigin);
+        html = injectOfflineFullNavigation(html);
+
+        const htmlPath = pageOutputPath(outRoot, pageUrl);
+        await writeFileSafe(htmlPath, html);
+      } catch (e) {
+        const message = e?.message || String(e);
+        console.warn(`Failed: ${pageUrl.toString()} (${message})`);
+        if (message.includes('Target page, context or browser has been closed')) {
+          await relaunch('target closed during navigation');
+        }
+      } finally {
+        page.off('response', onResponse);
+        try {
+          await page.close();
+        } catch {}
+      }
+    }
+  } finally {
+    try {
+      await context.close();
+    } catch {}
+    try {
+      await browser.close();
+    } catch {}
+  }
+
+  console.log('');
+  console.log(`Done. Pages saved: ${visitedPages.size}, Assets saved: ${savedAssets.size}`);
+  console.log(`Output: ${outRoot}`);
+}

package/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "site-mirror",
+  "version": "1.0.0",
+  "description": "CLI tool to mirror websites for offline browsing using Playwright",
+  "type": "module",
+  "bin": {
+    "site-mirror": "bin/cli.mjs"
+  },
+  "scripts": {
+    "postinstall": "node ./bin/postinstall.mjs"
+  },
+  "files": [
+    "bin/",
+    "lib/",
+    "README.md",
+    "LICENSE"
+  ],
+  "keywords": [
+    "mirror",
+    "offline",
+    "website",
+    "crawler",
+    "playwright",
+    "scraper",
+    "web-scraping",
+    "archiver",
+    "static-site"
+  ],
+  "author": "Mahesh Doiphode",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/MaheshDoiphode/site-mirror.git"
+  },
+  "bugs": {
+    "url": "https://github.com/MaheshDoiphode/site-mirror/issues"
+  },
+  "homepage": "https://github.com/MaheshDoiphode/site-mirror#readme",
+  "dependencies": {
+    "cheerio": "^1.0.0-rc.12",
+    "playwright": "^1.49.1"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  }
+}