web-snapshot-cli 0.2.0__tar.gz → 1.0.0__tar.gz

{web_snapshot_cli-0.2.0 → web_snapshot_cli-1.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: web-snapshot-cli
-Version: 0.2.0
+Version: 1.0.0
 Summary: Take offline snapshots of any website and restore them locally
 Project-URL: Homepage, https://github.com/codingsushi79/Snapshot
 Project-URL: Repository, https://github.com/codingsushi79/Snapshot
@@ -8,7 +8,7 @@ Author: snapshot contributors
 License-Expression: MIT
 License-File: LICENSE
 Keywords: cli,crawler,offline,snapshot,web-archive
-Classifier: Development Status :: 4 - Beta
+Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
@@ -85,6 +85,16 @@ snapshot --cookie session=abc123 --header "Authorization: Bearer TOKEN" \
 snapshot --crawl --sitemap https://example.com ./mirror --max-pages 500
 ```
 
+### Gobuster-style path discovery
+
+Brute-force common paths (including `robots.txt`, `sitemap.xml`, admin panels, etc.) using bundled wordlists (`common` + ~18k `large` entries). Add your own SecLists/gobuster wordlists with `-w`:
+
+```bash
+snapshot --crawl --gobuster https://example.com ./mirror
+snapshot --crawl -w common -w large -w /path/to/SecLists/Discovery/Web-Content/common.txt URL ./out
+snapshot --crawl --gobuster --wordlist-ext html,php,asp URL ./out
+```
+
 ### Resume an interrupted snapshot
 
 ```bash
@@ -137,6 +147,9 @@ snapshot -restore ./mirror --port 3000 --no-open
 | `--robots` / `--no-robots` | Respect robots.txt (default: on) |
 | `--crawl-delay` | Seconds to wait after each request (default: 0) |
 | `--sitemap` | Seed crawl from sitemap.xml |
+| `--gobuster`, `-g` | Discover paths with bundled wordlists |
+| `--wordlist`, `-w` | Custom or builtin wordlist (`common`, `large`) |
+| `--wordlist-ext` | Extensions to append per word (`html,php`) |
 | `--resume` | Skip pages/assets already saved |
 | `--verbose`, `-v` | Detailed log output |
 | `--dry-run` | Fetch without writing files |

{web_snapshot_cli-0.2.0 → web_snapshot_cli-1.0.0}/README.md

@@ -56,6 +56,16 @@ snapshot --cookie session=abc123 --header "Authorization: Bearer TOKEN" \
 snapshot --crawl --sitemap https://example.com ./mirror --max-pages 500
 ```
 
+### Gobuster-style path discovery
+
+Brute-force common paths (including `robots.txt`, `sitemap.xml`, admin panels, etc.) using bundled wordlists (`common` + ~18k `large` entries). Add your own SecLists/gobuster wordlists with `-w`:
+
+```bash
+snapshot --crawl --gobuster https://example.com ./mirror
+snapshot --crawl -w common -w large -w /path/to/SecLists/Discovery/Web-Content/common.txt URL ./out
+snapshot --crawl --gobuster --wordlist-ext html,php,asp URL ./out
+```
+
 ### Resume an interrupted snapshot
 
 ```bash
@@ -108,6 +118,9 @@ snapshot -restore ./mirror --port 3000 --no-open
 | `--robots` / `--no-robots` | Respect robots.txt (default: on) |
 | `--crawl-delay` | Seconds to wait after each request (default: 0) |
 | `--sitemap` | Seed crawl from sitemap.xml |
+| `--gobuster`, `-g` | Discover paths with bundled wordlists |
+| `--wordlist`, `-w` | Custom or builtin wordlist (`common`, `large`) |
+| `--wordlist-ext` | Extensions to append per word (`html,php`) |
 | `--resume` | Skip pages/assets already saved |
 | `--verbose`, `-v` | Detailed log output |
 | `--dry-run` | Fetch without writing files |

{web_snapshot_cli-0.2.0 → web_snapshot_cli-1.0.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "web-snapshot-cli"
-version = "0.2.0"
+version = "1.0.0"
 description = "Take offline snapshots of any website and restore them locally"
 readme = "README.md"
 license = "MIT"
@@ -12,7 +12,7 @@ requires-python = ">=3.10"
 authors = [{ name = "snapshot contributors" }]
 keywords = ["cli", "web-archive", "snapshot", "offline", "crawler"]
 classifiers = [
-    "Development Status :: 4 - Beta",
+    "Development Status :: 5 - Production/Stable",
     "Environment :: Console",
     "Intended Audience :: Developers",
     "License :: OSI Approved :: MIT License",
@@ -41,6 +41,10 @@ Repository = "https://github.com/codingsushi79/Snapshot"
 [tool.hatch.build.targets.wheel]
 packages = ["src/snapshot"]
 
+[tool.hatch.build.targets.wheel.force-include]
+"src/snapshot/wordlists/common.txt" = "snapshot/wordlists/common.txt"
+"src/snapshot/wordlists/large.txt" = "snapshot/wordlists/large.txt"
+
 [tool.pytest.ini_options]
 asyncio_mode = "auto"
 testpaths = ["tests"]

{web_snapshot_cli-0.2.0 → web_snapshot_cli-1.0.0}/src/snapshot/__init__.py

@@ -1,3 +1,3 @@
 """snapshot — offline website snapshots."""
 
-__version__ = "0.2.0"
+__version__ = "1.0.0"

{web_snapshot_cli-0.2.0 → web_snapshot_cli-1.0.0}/src/snapshot/cli.py

@@ -68,13 +68,20 @@ def _build_options(
     respect_robots: bool,
     crawl_delay: float,
     use_sitemap: bool,
+    gobuster: bool,
+    wordlists: tuple[str, ...],
+    wordlist_ext: str,
     resume: bool,
     verbose: bool,
     dry_run: bool,
 ) -> SnapshotOptions:
     default_ua = SnapshotOptions().user_agent
+    use_gobuster = gobuster or bool(extras.get("gobuster")) or bool(
+        wordlists or extras.get("wordlist")
+    )
+    crawl_enabled = crawl or bool(extras.get("crawl")) or use_gobuster
     return SnapshotOptions(
-        crawl=crawl or bool(extras.get("crawl")),
+        crawl=crawl_enabled,
         max_pages=int(extras.get("max_pages", max_pages)),
         max_depth=int(extras.get("depth", depth)),
         lang=str(extras.get("lang", lang)),
@@ -90,6 +97,9 @@ def _build_options(
         respect_robots=bool(extras.get("respect_robots", respect_robots)),
         crawl_delay=float(extras.get("crawl_delay", crawl_delay)),
         use_sitemap=use_sitemap or bool(extras.get("sitemap")),
+        gobuster=gobuster or bool(extras.get("gobuster")),
+        wordlists=list(wordlists) + _parse_list_value(extras.get("wordlist")),
+        wordlist_extensions=_parse_list_value(extras.get("wordlist_ext") or wordlist_ext),
         resume=resume or bool(extras.get("resume")),
         verbose=verbose or bool(extras.get("verbose")),
         dry_run=dry_run or bool(extras.get("dry_run")),
@@ -184,6 +194,23 @@ def _build_options(
     is_flag=True,
     help="Seed crawl URLs from sitemap.xml and robots.txt Sitemap directives.",
 )
+@click.option(
+    "--gobuster",
+    "-g",
+    is_flag=True,
+    help="Brute-force discover paths using bundled wordlists (common + large).",
+)
+@click.option(
+    "--wordlist",
+    "-w",
+    multiple=True,
+    help="Wordlist file or builtin name: common, large (repeatable). Enables path scanning.",
+)
+@click.option(
+    "--wordlist-ext",
+    default="",
+    help="Comma-separated extensions to try with each word (e.g. html,php,asp).",
+)
 @click.option(
     "--resume",
     is_flag=True,
@@ -222,6 +249,9 @@ def main(
     robots: bool,
     crawl_delay: float,
     use_sitemap: bool,
+    gobuster: bool,
+    wordlist: tuple[str, ...],
+    wordlist_ext: str,
     resume: bool,
     verbose: bool,
     dry_run: bool,
@@ -235,7 +265,8 @@ def main(
     Examples:
       snapshot https://example.com ./mirror
       snapshot --crawl https://docs.example.com ./docs --max-pages=200
-      snapshot --crawl --sitemap --crawl-delay 1 https://example.com ./mirror
+      snapshot --crawl --gobuster --sitemap https://example.com ./mirror
+      snapshot --crawl -w common -w /path/to/SecLists.txt URL ./out
       snapshot --cookie session=abc --header "Authorization: Bearer x" URL ./out
       snapshot --crawl --include '/blog/*' --exclude '/blog/drafts/*' URL ./out
       snapshot --resume --crawl URL ./mirror
@@ -270,6 +301,9 @@ def main(
         respect_robots=robots,
         crawl_delay=crawl_delay,
         use_sitemap=use_sitemap,
+        gobuster=gobuster,
+        wordlists=wordlist,
+        wordlist_ext=wordlist_ext,
         resume=resume,
         verbose=verbose,
         dry_run=dry_run,
@@ -283,6 +317,9 @@ def main(
         console.print(f"  crawl: up to {options.max_pages} pages, depth {options.max_depth}")
     if options.use_sitemap:
         console.print("  sitemap: enabled")
+    if options.gobuster or options.wordlists:
+        sources = options.wordlists or ["common", "large"]
+        console.print(f"  wordlist: {', '.join(sources)}")
     if options.resume:
         console.print("  resume: enabled")
     console.print(f"  format: {options.lang}")
@@ -314,6 +351,8 @@ def main(
         console.print(
             f"  skipped: {result.pages_skipped} pages, {result.assets_skipped} assets (resume)"
         )
+    if result.wordlist_hits:
+        console.print(f"  wordlist: {result.wordlist_hits} paths discovered")
     if result.errors:
         console.print(f"[yellow]{len(result.errors)} errors[/yellow] (see below)")
         for err in result.errors[:10]:

{web_snapshot_cli-0.2.0 → web_snapshot_cli-1.0.0}/src/snapshot/downloader.py

@@ -13,6 +13,7 @@ from markdownify import markdownify
 from snapshot.crawl_policy import (
     RobotsChecker,
     discover_sitemap_urls,
+    parse_sitemap_xml,
     url_matches_filters,
 )
 from snapshot.manifest import MANIFEST_NAME, SnapshotManifest
@@ -27,6 +28,7 @@ from snapshot.utils import (
     same_origin,
     url_to_local_path,
 )
+from snapshot.wordlist import load_words, resolve_wordlist_sources, scan_wordlists
 
 
 @dataclass
@@ -39,7 +41,7 @@ class SnapshotOptions:
     timeout: float = 15.0
     concurrency: int = 16
     same_origin_only: bool = True
-    user_agent: str = "web-snapshot-cli/0.2 (+https://github.com/codingsushi79/Snapshot)"
+    user_agent: str = "web-snapshot-cli/1.0 (+https://github.com/codingsushi79/Snapshot)"
     cookies: list[str] = field(default_factory=list)
     headers: list[str] = field(default_factory=list)
     include_patterns: list[str] = field(default_factory=list)
@@ -47,6 +49,9 @@ class SnapshotOptions:
     respect_robots: bool = True
     crawl_delay: float = 0.0
     use_sitemap: bool = False
+    gobuster: bool = False
+    wordlists: list[str] = field(default_factory=list)
+    wordlist_extensions: list[str] = field(default_factory=list)
     resume: bool = False
     verbose: bool = False
     dry_run: bool = False
@@ -58,6 +63,7 @@ class SnapshotResult:
     assets_saved: int = 0
     pages_skipped: int = 0
     assets_skipped: int = 0
+    wordlist_hits: int = 0
     errors: list[str] = field(default_factory=list)
 
 
@@ -75,6 +81,7 @@ class SnapshotEngine:
         self._robots = RobotsChecker(options.user_agent, options.respect_robots)
         self._pages_skipped = 0
         self._assets_skipped = 0
+        self._wordlist_hits = 0
         self._log_fn: Callable[[str], None] | None = None
 
     async def run(
@@ -119,6 +126,7 @@ class SnapshotEngine:
             assets_saved=len(self._seen_assets),
             pages_skipped=self._pages_skipped,
             assets_skipped=self._assets_skipped,
+            wordlist_hits=self._wordlist_hits,
             errors=self._errors,
         )
 
@@ -133,6 +141,8 @@ class SnapshotEngine:
             "respect_robots": self.options.respect_robots,
             "crawl_delay": self.options.crawl_delay,
             "use_sitemap": self.options.use_sitemap,
+            "gobuster": self.options.gobuster,
+            "wordlists": self.options.wordlists,
         }
 
     def _build_headers(self) -> dict[str, str]:
@@ -198,6 +208,35 @@ class SnapshotEngine:
             self._log(f"sitemap: discovered {len(sitemap_urls)} URLs")
             seed_urls.extend(sitemap_urls)
 
+        wordlist_sources = resolve_wordlist_sources(
+            self.options.gobuster,
+            self.options.wordlists,
+        )
+        if wordlist_sources:
+            if progress:
+                progress("loading wordlists…")
+            words = load_words(wordlist_sources)
+            extensions = tuple(self.options.wordlist_extensions)
+            found = await scan_wordlists(
+                self._client,
+                self.root_url,
+                words,
+                extensions=extensions,
+                concurrency=self.options.concurrency,
+                crawl_delay=self.options.crawl_delay,
+                robots=self._robots,
+                url_allowed=self._url_allowed,
+                log=self._log,
+                progress=progress,
+            )
+            self._wordlist_hits = len(found)
+            extra = await self._expand_metadata_urls(found)
+            seed_urls.extend(found)
+            seed_urls.extend(extra)
+            self._log(
+                f"wordlist: {len(found)} paths, {len(extra)} extra URLs from robots/sitemaps"
+            )
+
         for url in seed_urls:
             normalized = normalize_url(url)
             if not normalized or not self._url_allowed(normalized):
@@ -216,6 +255,35 @@ class SnapshotEngine:
             worker.cancel()
         await asyncio.gather(*workers, return_exceptions=True)
 
+    async def _expand_metadata_urls(self, discovered: list[str]) -> list[str]:
+        """Parse robots.txt and sitemap XML found during wordlist scanning."""
+        from snapshot.crawl_policy import parse_robots_sitemaps
+
+        extra: list[str] = []
+        seen: set[str] = set(discovered)
+        for url in discovered:
+            lower = url.lower()
+            try:
+                if lower.endswith("/robots.txt") or lower.endswith("robots.txt"):
+                    response = await self._client.get(url)
+                    if response.status_code == 200:
+                        for sitemap_url in parse_robots_sitemaps(response.text):
+                            if sitemap_url not in seen:
+                                seen.add(sitemap_url)
+                                extra.append(sitemap_url)
+                if "sitemap" in lower and (
+                    lower.endswith(".xml") or lower.endswith(".xml.gz") or lower.endswith(".txt")
+                ):
+                    response = await self._client.get(url)
+                    if response.status_code == 200:
+                        for page_url in parse_sitemap_xml(response.text):
+                            if page_url not in seen:
+                                seen.add(page_url)
+                                extra.append(page_url)
+            except Exception as exc:  # noqa: BLE001
+                self._errors.append(f"{url}: {exc}")
+        return extra
+
     async def _crawl_worker(
         self,
         queue: asyncio.Queue[tuple[str, int]],

web_snapshot_cli-1.0.0/src/snapshot/wordlist.py

@@ -0,0 +1,171 @@
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Callable
+from importlib import resources
+from pathlib import Path
+from urllib.parse import urljoin, urlparse
+
+import httpx
+
+from snapshot.crawl_policy import RobotsChecker
+
+BUILTIN_WORDLISTS = ("common", "large")
+HIT_STATUS = {200, 201, 204, 301, 302, 307, 308, 401, 403}
+
+
+def resolve_wordlist_sources(
+    gobuster: bool,
+    wordlist_args: list[str],
+) -> list[Path | str]:
+    """Return paths and/or builtin names to load."""
+    if wordlist_args:
+        return list(wordlist_args)
+    if gobuster:
+        return list(BUILTIN_WORDLISTS)
+    return []
+
+
+def load_words(sources: list[Path | str]) -> list[str]:
+    words: list[str] = []
+    seen: set[str] = set()
+    for source in sources:
+        for word in _load_source(source):
+            if word not in seen:
+                seen.add(word)
+                words.append(word)
+    return words
+
+
+def _load_source(source: Path | str) -> list[str]:
+    name = str(source).strip()
+    if name in BUILTIN_WORDLISTS:
+        return _load_builtin(name)
+    path = Path(name)
+    if not path.is_file():
+        msg = f"wordlist not found: {name} (use common, large, or a file path)"
+        raise FileNotFoundError(msg)
+    return _parse_wordlist_text(path.read_text(encoding="utf-8", errors="replace"))
+
+
+def _load_builtin(name: str) -> list[str]:
+    text = resources.files("snapshot.wordlists").joinpath(f"{name}.txt").read_text(
+        encoding="utf-8"
+    )
+    return _parse_wordlist_text(text)
+
+
+def _parse_wordlist_text(text: str) -> list[str]:
+    words: list[str] = []
+    for line in text.splitlines():
+        stripped = line.strip()
+        if not stripped or stripped.startswith("#"):
+            continue
+        words.append(stripped.lstrip("/"))
+    return words
+
+
+def paths_for_word(word: str, extensions: tuple[str, ...]) -> list[str]:
+    """Build URL path candidates for a single wordlist entry."""
+    cleaned = word.strip().strip("/")
+    if not cleaned:
+        return []
+
+    basename = cleaned.split("/")[-1]
+    paths = [f"/{cleaned}"]
+
+    if "." not in basename:
+        paths.append(f"/{cleaned}/")
+        for ext in extensions:
+            suffix = ext if ext.startswith(".") else f".{ext}"
+            paths.append(f"/{cleaned}{suffix}")
+
+    return list(dict.fromkeys(paths))
+
+
+async def scan_wordlists(
+    client: httpx.AsyncClient,
+    root_url: str,
+    words: list[str],
+    *,
+    extensions: tuple[str, ...],
+    concurrency: int,
+    crawl_delay: float,
+    robots: RobotsChecker,
+    url_allowed: Callable[[str], bool],
+    log: Callable[[str], None] | None = None,
+    progress: Callable[[str], None] | None = None,
+) -> list[str]:
+    """Probe wordlist paths and return URLs that appear to exist."""
+    parsed_root = urlparse(root_url)
+    base = f"{parsed_root.scheme}://{parsed_root.netloc}"
+
+    candidates: list[str] = []
+    seen_candidates: set[str] = set()
+    for word in words:
+        for path in paths_for_word(word, extensions):
+            absolute = normalize_probe_url(base, path)
+            if absolute is None or absolute in seen_candidates:
+                continue
+            if not url_allowed(absolute):
+                continue
+            seen_candidates.add(absolute)
+            candidates.append(absolute)
+
+    if not candidates:
+        return []
+
+    if progress:
+        progress(f"wordlist: probing {len(candidates)} paths…")
+    if log:
+        log(f"wordlist: probing {len(candidates)} paths from {len(words)} words")
+
+    sem = asyncio.Semaphore(max(1, concurrency))
+    found: list[str] = []
+    found_lock = asyncio.Lock()
+    probed = 0
+
+    async def probe(target: str) -> None:
+        nonlocal probed
+        if robots.enabled and not await robots.can_fetch(client, target):
+            return
+        async with sem:
+            hit = await _probe_url(client, target)
+            if crawl_delay > 0:
+                await asyncio.sleep(crawl_delay)
+        probed += 1
+        if probed % 250 == 0 and progress:
+            progress(f"wordlist: {probed}/{len(candidates)} probes, {len(found)} hits")
+        if hit:
+            async with found_lock:
+                found.append(target)
+            if log:
+                log(f"wordlist hit: {target}")
+
+    await asyncio.gather(*(probe(url) for url in candidates))
+
+    if log:
+        log(f"wordlist: {len(found)} paths found")
+    return sorted(found)
+
+
+def normalize_probe_url(base: str, path: str) -> str | None:
+    joined = urljoin(f"{base}/", path.lstrip("/"))
+    parsed = urlparse(joined)
+    if not parsed.scheme or not parsed.netloc:
+        return None
+    return parsed._replace(fragment="").geturl()
+
+
+async def _probe_url(client: httpx.AsyncClient, url: str) -> bool:
+    for method in ("HEAD", "GET"):
+        try:
+            response = await client.request(method, url, follow_redirects=False)
+        except httpx.RequestError:
+            return False
+        if response.status_code in HIT_STATUS:
+            return True
+        if response.status_code == 405 and method == "HEAD":
+            continue
+        return False
+    return False

web_snapshot_cli-1.0.0/src/snapshot/wordlists/common.txt

@@ -0,0 +1,132 @@
+# Essential discovery paths (robots, sitemaps, well-known, admin)
+robots.txt
+sitemap.xml
+sitemap_index.xml
+sitemap-index.xml
+sitemap1.xml
+sitemap.txt
+sitemap.gz
+sitemap.xml.gz
+sitemap_index.xml.gz
+atom.xml
+feed
+rss
+rss.xml
+feed.xml
+index.xml
+crossdomain.xml
+clientaccesspolicy.xml
+favicon.ico
+apple-touch-icon.png
+.well-known/security.txt
+.well-known/change-password
+.well-known/assetlinks.json
+.well-known/apple-app-site-association
+.well-known/ai-plugin.json
+.well-known/openid-configuration
+.well-known/jwks.json
+humans.txt
+ads.txt
+app-ads.txt
+security.txt
+browserconfig.xml
+manifest.json
+site.webmanifest
+opensearch.xml
+health
+healthz
+healthcheck
+status
+ping
+version
+api
+api/v1
+api/v2
+graphql
+swagger
+swagger.json
+swagger-ui
+swagger-ui.html
+openapi.json
+docs
+documentation
+redoc
+admin
+administrator
+login
+logout
+signin
+signup
+register
+auth
+oauth
+dashboard
+portal
+console
+panel
+cpanel
+webmail
+phpmyadmin
+wp-admin
+wp-login.php
+wp-json
+xmlrpc.php
+.git
+.git/config
+.git/HEAD
+.env
+.env.local
+.env.production
+backup
+backups
+old
+test
+dev
+staging
+tmp
+temp
+upload
+uploads
+files
+assets
+static
+public
+media
+images
+img
+css
+js
+javascript
+dist
+build
+cache
+config
+settings
+search
+archive
+archives
+blog
+news
+press
+about
+about-us
+contact
+contact-us
+help
+support
+faq
+privacy
+terms
+legal
+sitemap
+sitemaps
+sitemap-index
+sitemap_index
+sitemapindex
+sitemap.php
+sitemap.html
+sitemap.asp
+sitemap.aspx
+robots
+robots.php
+robots.html