web-snapshot-cli 0.2.0__tar.gz → 1.1.0__tar.gz

web_snapshot_cli-1.1.0/.github/workflows/pages.yml

@@ -0,0 +1,33 @@
+name: Deploy GitHub Pages
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - "docs/**"
+      - ".github/workflows/pages.yml"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: pages
+  cancel-in-progress: false
+
+jobs:
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/configure-pages@v5
+      - uses: actions/upload-pages-artifact@v3
+        with:
+          path: docs
+      - uses: actions/deploy-pages@v4
+        id: deployment

{web_snapshot_cli-0.2.0 → web_snapshot_cli-1.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: web-snapshot-cli
-Version: 0.2.0
+Version: 1.1.0
 Summary: Take offline snapshots of any website and restore them locally
 Project-URL: Homepage, https://github.com/codingsushi79/Snapshot
 Project-URL: Repository, https://github.com/codingsushi79/Snapshot
@@ -8,7 +8,7 @@ Author: snapshot contributors
 License-Expression: MIT
 License-File: LICENSE
 Keywords: cli,crawler,offline,snapshot,web-archive
-Classifier: Development Status :: 4 - Beta
+Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
@@ -45,6 +45,8 @@ Then run:
 snapshot https://example.com ./mirror
 ```
 
+**Command generator:** [snapshot.sushii.dev](https://snapshot.sushii.dev/)
+
 ## Usage
 
 ### Snapshot a single page
@@ -85,6 +87,16 @@ snapshot --cookie session=abc123 --header "Authorization: Bearer TOKEN" \
 snapshot --crawl --sitemap https://example.com ./mirror --max-pages 500
 ```
 
+### Gobuster-style path discovery
+
+Brute-force common paths (including `robots.txt`, `sitemap.xml`, admin panels, etc.) using bundled wordlists (`common` + ~18k `large` entries). Add your own SecLists/gobuster wordlists with `-w`:
+
+```bash
+snapshot --crawl --gobuster https://example.com ./mirror
+snapshot --crawl -w common -w large -w /path/to/SecLists/Discovery/Web-Content/common.txt URL ./out
+snapshot --crawl --gobuster --wordlist-ext html,php,asp URL ./out
+```
+
 ### Resume an interrupted snapshot
 
 ```bash
@@ -137,6 +149,9 @@ snapshot -restore ./mirror --port 3000 --no-open
 | `--robots` / `--no-robots` | Respect robots.txt (default: on) |
 | `--crawl-delay` | Seconds to wait after each request (default: 0) |
 | `--sitemap` | Seed crawl from sitemap.xml |
+| `--gobuster`, `-g` | Discover paths with bundled wordlists |
+| `--wordlist`, `-w` | Custom or builtin wordlist (`common`, `large`) |
+| `--wordlist-ext` | Extensions to append per word (`html,php`) |
 | `--resume` | Skip pages/assets already saved |
 | `--verbose`, `-v` | Detailed log output |
 | `--dry-run` | Fetch without writing files |

{web_snapshot_cli-0.2.0 → web_snapshot_cli-1.1.0}/README.md

@@ -16,6 +16,8 @@ Then run:
 snapshot https://example.com ./mirror
 ```
 
+**Command generator:** [snapshot.sushii.dev](https://snapshot.sushii.dev/)
+
 ## Usage
 
 ### Snapshot a single page
@@ -56,6 +58,16 @@ snapshot --cookie session=abc123 --header "Authorization: Bearer TOKEN" \
 snapshot --crawl --sitemap https://example.com ./mirror --max-pages 500
 ```
 
+### Gobuster-style path discovery
+
+Brute-force common paths (including `robots.txt`, `sitemap.xml`, admin panels, etc.) using bundled wordlists (`common` + ~18k `large` entries). Add your own SecLists/gobuster wordlists with `-w`:
+
+```bash
+snapshot --crawl --gobuster https://example.com ./mirror
+snapshot --crawl -w common -w large -w /path/to/SecLists/Discovery/Web-Content/common.txt URL ./out
+snapshot --crawl --gobuster --wordlist-ext html,php,asp URL ./out
+```
+
 ### Resume an interrupted snapshot
 
 ```bash
@@ -108,6 +120,9 @@ snapshot -restore ./mirror --port 3000 --no-open
 | `--robots` / `--no-robots` | Respect robots.txt (default: on) |
 | `--crawl-delay` | Seconds to wait after each request (default: 0) |
 | `--sitemap` | Seed crawl from sitemap.xml |
+| `--gobuster`, `-g` | Discover paths with bundled wordlists |
+| `--wordlist`, `-w` | Custom or builtin wordlist (`common`, `large`) |
+| `--wordlist-ext` | Extensions to append per word (`html,php`) |
 | `--resume` | Skip pages/assets already saved |
 | `--verbose`, `-v` | Detailed log output |
 | `--dry-run` | Fetch without writing files |

web_snapshot_cli-1.1.0/docs/CNAME

@@ -0,0 +1 @@
+snapshot.sushii.dev

web_snapshot_cli-1.1.0/docs/app.js

@@ -0,0 +1,221 @@
+const $ = (sel) => document.querySelector(sel);
+const $$ = (sel) => document.querySelectorAll(sel);
+
+const PRESETS = {
+  quick: {
+    crawl: false,
+    gobuster: false,
+    sitemap: false,
+    maxPages: 50,
+    depth: 3,
+    crawlDelay: 0,
+  },
+  crawl: {
+    crawl: true,
+    gobuster: false,
+    sitemap: true,
+    maxPages: 200,
+    depth: 5,
+    crawlDelay: 0,
+    resume: true,
+    verbose: true,
+  },
+  full: {
+    crawl: true,
+    gobuster: true,
+    sitemap: true,
+    maxPages: 5000,
+    depth: 10,
+    crawlDelay: 0.5,
+    wordlistExt: "html,php,asp,aspx",
+    resume: true,
+    verbose: true,
+  },
+};
+
+function shellQuote(value) {
+  if (!value) return '""';
+  if (/^[A-Za-z0-9_./~-]+$/.test(value)) return value;
+  return `"${value.replace(/"/g, '\\"')}"`;
+}
+
+function linesFromTextarea(id) {
+  const el = $(id);
+  if (!el || !el.value.trim()) return [];
+  return el.value
+    .split("\n")
+    .map((l) => l.trim())
+    .filter(Boolean);
+}
+
+function isRestoreMode() {
+  return document.querySelector(".mode-tab.active")?.dataset.mode === "restore";
+}
+
+function buildCommand() {
+  if (isRestoreMode()) {
+    return buildRestoreCommand();
+  }
+  return buildSnapshotCommand();
+}
+
+function buildRestoreCommand() {
+  const parts = ["snapshot", "-restore"];
+  const dir = $("#outputDir").value.trim() || "./mirror";
+  const port = $("#port").value;
+  const host = $("#host").value.trim();
+  const noOpen = $("#noOpen").checked;
+
+  parts.push(shellQuote(dir));
+  if (port && port !== "8080") parts.push("--port", port);
+  if (host && host !== "127.0.0.1") parts.push("--host", shellQuote(host));
+  if (noOpen) parts.push("--no-open");
+
+  return parts.join(" ");
+}
+
+function buildSnapshotCommand() {
+  const parts = ["snapshot"];
+  const flags = [];
+
+  if ($("#crawl").checked) flags.push("--crawl");
+  if ($("#gobuster").checked) flags.push("--gobuster");
+  if ($("#sitemap").checked) flags.push("--sitemap");
+  if ($("#resume").checked) flags.push("--resume");
+  if ($("#verbose").checked) flags.push("--verbose");
+  if ($("#dryRun").checked) flags.push("--dry-run");
+  if ($("#noAssets").checked) flags.push("--no-assets");
+  if (!$("#robots").checked) flags.push("--no-robots");
+  if (!$("#sameOrigin").checked) flags.push("--no-same-origin");
+
+  const lang = $("#lang").value;
+  if (lang && lang !== "html") flags.push("--lang", lang);
+
+  const maxPages = $("#maxPages").value;
+  if (maxPages && maxPages !== "50") flags.push("--max-pages", maxPages);
+
+  const depth = $("#depth").value;
+  if (depth && depth !== "3") flags.push("--depth", depth);
+
+  const timeout = $("#timeout").value;
+  if (timeout && timeout !== "15") flags.push("--timeout", timeout);
+
+  const concurrency = $("#concurrency").value;
+  if (concurrency && concurrency !== "16") flags.push("--concurrency", concurrency);
+
+  const crawlDelay = $("#crawlDelay").value;
+  if (crawlDelay && crawlDelay !== "0" && crawlDelay !== "0.0") {
+    flags.push("--crawl-delay", crawlDelay);
+  }
+
+  const userAgent = $("#userAgent").value.trim();
+  if (userAgent) flags.push("--user-agent", shellQuote(userAgent));
+
+  const wordlistExt = $("#wordlistExt").value.trim();
+  if (wordlistExt) flags.push("--wordlist-ext", shellQuote(wordlistExt));
+
+  for (const cookie of linesFromTextarea("#cookies")) {
+    flags.push("--cookie", shellQuote(cookie));
+  }
+  for (const header of linesFromTextarea("#headers")) {
+    flags.push("--header", shellQuote(header));
+  }
+  for (const inc of linesFromTextarea("#includes")) {
+    flags.push("--include", shellQuote(inc));
+  }
+  for (const exc of linesFromTextarea("#excludes")) {
+    flags.push("--exclude", shellQuote(exc));
+  }
+  for (const wl of linesFromTextarea("#wordlists")) {
+    flags.push("--wordlist", shellQuote(wl));
+  }
+
+  parts.push(...flags);
+
+  let url = $("#url").value.trim();
+  if (url && !/^https?:\/\//i.test(url)) url = `https://${url}`;
+  const out = $("#outputDir").value.trim() || "./mirror";
+
+  parts.push(shellQuote(url || "https://example.com"));
+  parts.push(shellQuote(out));
+
+  return parts.join(" ");
+}
+
+function updateCommand() {
+  const cmd = buildCommand();
+  $("#command").textContent = cmd;
+
+  const restoreHint = $("#restoreHint");
+  if (isRestoreMode()) {
+    restoreHint.classList.add("hidden");
+  } else {
+    restoreHint.classList.remove("hidden");
+    const dir = $("#outputDir").value.trim() || "./mirror";
+    $("#restoreCommand").textContent = `snapshot -restore ${shellQuote(dir)}`;
+  }
+}
+
+function applyPreset(name) {
+  const p = PRESETS[name];
+  if (!p) return;
+
+  $("#crawl").checked = !!p.crawl;
+  $("#gobuster").checked = !!p.gobuster;
+  $("#sitemap").checked = !!p.sitemap;
+  $("#resume").checked = !!p.resume;
+  $("#verbose").checked = !!p.verbose;
+  if (p.maxPages != null) $("#maxPages").value = p.maxPages;
+  if (p.depth != null) $("#depth").value = p.depth;
+  if (p.crawlDelay != null) $("#crawlDelay").value = p.crawlDelay;
+  if (p.wordlistExt != null) $("#wordlistExt").value = p.wordlistExt;
+
+  setMode("snapshot");
+  updateCommand();
+}
+
+function setMode(mode) {
+  $$(".mode-tab").forEach((tab) => {
+    tab.classList.toggle("active", tab.dataset.mode === mode);
+  });
+
+  const snapshotOnly = $$(".snapshot-only");
+  const restoreOnly = $$(".restore-only");
+
+  snapshotOnly.forEach((el) => el.classList.toggle("hidden", mode === "restore"));
+  restoreOnly.forEach((el) => el.classList.toggle("hidden", mode !== "restore"));
+
+  updateCommand();
+}
+
+async function copyCommand() {
+  const text = $("#command").textContent;
+  await navigator.clipboard.writeText(text);
+  const btn = $("#copyBtn");
+  btn.textContent = "Copied!";
+  btn.classList.add("copied");
+  setTimeout(() => {
+    btn.textContent = "Copy";
+    btn.classList.remove("copied");
+  }, 1500);
+}
+
+function init() {
+  $$(".mode-tab").forEach((tab) => {
+    tab.addEventListener("click", () => setMode(tab.dataset.mode));
+  });
+
+  $$(".preset-btn").forEach((btn) => {
+    btn.addEventListener("click", () => applyPreset(btn.dataset.preset));
+  });
+
+  $("#copyBtn").addEventListener("click", copyCommand);
+
+  document.addEventListener("input", updateCommand);
+  document.addEventListener("change", updateCommand);
+
+  setMode("snapshot");
+  updateCommand();
+}
+
+document.addEventListener("DOMContentLoaded", init);

web_snapshot_cli-1.1.0/docs/index.html

@@ -0,0 +1,200 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <title>snapshot — command generator</title>
+  <meta name="description" content="Build snapshot CLI commands for offline website mirrors." />
+  <link rel="stylesheet" href="style.css" />
+</head>
+<body>
+  <header>
+    <h1>snapshot command generator</h1>
+    <p>
+      Build install and run commands for
+      <a href="https://github.com/codingsushi79/Snapshot">web-snapshot-cli</a>
+    </p>
+  </header>
+
+  <div class="layout">
+    <div class="panel">
+      <div class="mode-tabs">
+        <button type="button" class="mode-tab active" data-mode="snapshot">Snapshot</button>
+        <button type="button" class="mode-tab" data-mode="restore">Restore</button>
+      </div>
+
+      <div class="presets snapshot-only">
+        <button type="button" class="preset-btn" data-preset="quick">Single page</button>
+        <button type="button" class="preset-btn" data-preset="crawl">Crawl</button>
+        <button type="button" class="preset-btn" data-preset="full">Full + gobuster</button>
+      </div>
+
+      <h2>Target</h2>
+      <div class="field snapshot-only">
+        <label for="url">Website URL</label>
+        <input type="url" id="url" placeholder="https://example.com" value="https://example.com" />
+      </div>
+      <div class="field">
+        <label for="outputDir">Output directory</label>
+        <input type="text" id="outputDir" placeholder="./mirror or ~/sites/example" value="./mirror" />
+        <div class="hint">Where files are saved (snapshot) or served from (restore)</div>
+      </div>
+
+      <hr class="section-divider snapshot-only" />
+
+      <h2 class="snapshot-only">Discovery</h2>
+      <div class="check-grid snapshot-only">
+        <label class="check-item">
+          <input type="checkbox" id="crawl" />
+          <div>--crawl<span>Follow same-origin links</span></div>
+        </label>
+        <label class="check-item">
+          <input type="checkbox" id="gobuster" />
+          <div>--gobuster<span>Bundled wordlists; soft 404 pages are filtered out</span></div>
+        </label>
+        <label class="check-item">
+          <input type="checkbox" id="sitemap" />
+          <div>--sitemap<span>Seed from sitemap.xml / robots.txt</span></div>
+        </label>
+        <label class="check-item">
+          <input type="checkbox" id="resume" />
+          <div>--resume<span>Skip already-downloaded files</span></div>
+        </label>
+      </div>
+
+      <div class="field snapshot-only">
+        <label for="wordlists">Extra wordlists (one per line)</label>
+        <textarea id="wordlists" placeholder="common&#10;large&#10;/path/to/SecLists/Discovery/Web-Content/common.txt"></textarea>
+        <div class="hint">Builtin names: <code>common</code>, <code>large</code>. --gobuster uses both by default. Pages that return 200 but show 404/not-found content are skipped.</div>
+      </div>
+      <div class="field snapshot-only">
+        <label for="wordlistExt">Wordlist extensions</label>
+        <input type="text" id="wordlistExt" placeholder="html,php,asp,aspx" />
+      </div>
+
+      <hr class="section-divider snapshot-only" />
+
+      <h2 class="snapshot-only">Crawl limits</h2>
+      <div class="row snapshot-only">
+        <div class="field">
+          <label for="maxPages">Max pages</label>
+          <input type="number" id="maxPages" value="50" min="1" />
+        </div>
+        <div class="field">
+          <label for="depth">Depth</label>
+          <input type="number" id="depth" value="3" min="1" />
+        </div>
+        <div class="field">
+          <label for="concurrency">Concurrency</label>
+          <input type="number" id="concurrency" value="16" min="1" />
+        </div>
+        <div class="field">
+          <label for="crawlDelay">Crawl delay (sec)</label>
+          <input type="number" id="crawlDelay" value="0" min="0" step="0.1" />
+        </div>
+        <div class="field">
+          <label for="timeout">Timeout (sec)</label>
+          <input type="number" id="timeout" value="15" min="1" step="0.5" />
+        </div>
+        <div class="field">
+          <label for="lang">Output format</label>
+          <select id="lang">
+            <option value="html">html</option>
+            <option value="md">md</option>
+          </select>
+        </div>
+      </div>
+
+      <hr class="section-divider snapshot-only" />
+
+      <h2 class="snapshot-only">Filters &amp; auth</h2>
+      <div class="field snapshot-only">
+        <label for="includes">Include globs (one per line)</label>
+        <textarea id="includes" placeholder="/docs/*&#10;/blog/*"></textarea>
+      </div>
+      <div class="field snapshot-only">
+        <label for="excludes">Exclude globs (one per line)</label>
+        <textarea id="excludes" placeholder="/admin/*&#10;/drafts/*"></textarea>
+      </div>
+      <div class="field snapshot-only">
+        <label for="cookies">Cookies (name=value, one per line)</label>
+        <textarea id="cookies" placeholder="session=abc123"></textarea>
+      </div>
+      <div class="field snapshot-only">
+        <label for="headers">Headers (Name: Value, one per line)</label>
+        <textarea id="headers" placeholder="Authorization: Bearer TOKEN"></textarea>
+      </div>
+      <div class="field snapshot-only">
+        <label for="userAgent">Custom User-Agent</label>
+        <input type="text" id="userAgent" placeholder="Leave blank for default" />
+      </div>
+
+      <hr class="section-divider" />
+
+      <h2>Options</h2>
+      <div class="check-grid snapshot-only">
+        <label class="check-item">
+          <input type="checkbox" id="robots" checked />
+          <div>--robots<span>Respect robots.txt</span></div>
+        </label>
+        <label class="check-item">
+          <input type="checkbox" id="sameOrigin" checked />
+          <div>--same-origin<span>Same origin only</span></div>
+        </label>
+        <label class="check-item">
+          <input type="checkbox" id="noAssets" />
+          <div>--no-assets<span>Skip CSS/JS/images</span></div>
+        </label>
+        <label class="check-item">
+          <input type="checkbox" id="verbose" />
+          <div>--verbose<span>Detailed logs</span></div>
+        </label>
+        <label class="check-item">
+          <input type="checkbox" id="dryRun" />
+          <div>--dry-run<span>Don't write files</span></div>
+        </label>
+      </div>
+      <div class="row restore-only hidden">
+        <div class="field">
+          <label for="port">Port</label>
+          <input type="number" id="port" value="8080" min="1" />
+        </div>
+        <div class="field">
+          <label for="host">Host</label>
+          <input type="text" id="host" value="127.0.0.1" />
+        </div>
+      </div>
+      <label class="check-item restore-only hidden">
+        <input type="checkbox" id="noOpen" />
+        <div>--no-open<span>Don't open browser</span></div>
+      </label>
+    </div>
+
+    <div class="panel output-panel">
+      <h2>Generated command</h2>
+      <div class="command-box">
+        <code id="command">snapshot https://example.com ./mirror</code>
+        <button type="button" class="copy-btn" id="copyBtn">Copy</button>
+      </div>
+
+      <div class="install-line">
+        Install first:
+        <code>pip install web-snapshot-cli</code>
+      </div>
+
+      <div class="install-line" id="restoreHint">
+        After snapshot, restore locally:
+        <code id="restoreCommand">snapshot -restore ./mirror</code>
+      </div>
+    </div>
+  </div>
+
+  <footer>
+  <a href="https://github.com/codingsushi79/Snapshot">GitHub</a>
+  ·
+  <a href="https://pypi.org/project/web-snapshot-cli/">PyPI</a>
+  </footer>
+
+  <script src="app.js"></script>
+</body>
+</html>