web-snapshot-cli 0.1.0__py3-none-any.whl

snapshot/utils.py

@@ -0,0 +1,153 @@
+from __future__ import annotations
+
+import hashlib
+import mimetypes
+import os
+import re
+from pathlib import Path, PurePosixPath
+from urllib.parse import urljoin, urlparse, urlunparse
+
+# Attributes that may contain fetchable URLs.
+URL_ATTRS = ("href", "src", "poster", "data-src", "data-background")
+SRCSET_ATTRS = ("srcset", "data-srcset")
+
+# Tags treated as HTML pages when crawled.
+PAGE_EXTENSIONS = {".html", ".htm", ".xhtml", ""}
+ASSET_EXTENSIONS = {
+    ".css",
+    ".js",
+    ".mjs",
+    ".json",
+    ".xml",
+    ".svg",
+    ".png",
+    ".jpg",
+    ".jpeg",
+    ".gif",
+    ".webp",
+    ".avif",
+    ".ico",
+    ".woff",
+    ".woff2",
+    ".ttf",
+    ".eot",
+    ".mp4",
+    ".webm",
+    ".mp3",
+    ".pdf",
+}
+
+SKIP_SCHEMES = {"mailto", "tel", "javascript", "data", "blob", "about", "file"}
+
+
+def normalize_url(url: str, base: str | None = None) -> str | None:
+    """Resolve and normalize a URL. Returns None for non-fetchable schemes."""
+    if not url or url.startswith("#"):
+        return None
+    url = url.strip()
+    if base:
+        url = urljoin(base, url)
+    parsed = urlparse(url)
+    if parsed.scheme and parsed.scheme.lower() in SKIP_SCHEMES:
+        return None
+    if not parsed.scheme:
+        return None
+    # Drop fragment; keep query for distinct resources.
+    normalized = parsed._replace(fragment="")
+    path = normalized.path or "/"
+    return urlunparse(normalized._replace(path=path))
+
+
+def same_origin(a: str, b: str) -> bool:
+    pa, pb = urlparse(a), urlparse(b)
+    return (pa.scheme, pa.netloc) == (pb.scheme, pb.netloc)
+
+
+def url_to_local_path(
+    url: str,
+    output_dir: Path,
+    page_url: str | None = None,
+    page_ext: str = ".html",
+) -> Path:
+    """Map a remote URL to a local filesystem path inside output_dir."""
+    parsed = urlparse(url)
+    host_dir = _safe_name(parsed.netloc)
+    path = PurePosixPath(parsed.path or "/")
+
+    if path.suffix:
+        rel = path.as_posix().lstrip("/")
+    elif _looks_like_page(url):
+        index_name = f"index{page_ext}"
+        posix = path.as_posix().lstrip("/")
+        if not posix or path.as_posix().endswith("/"):
+            rel = f"{posix}{index_name}" if posix else index_name
+        else:
+            rel = f"{posix}/{index_name}"
+    else:
+        digest = hashlib.sha1(url.encode()).hexdigest()[:10]
+        ext = _guess_extension(url) or ".bin"
+        rel = f"_assets/{digest}{ext}"
+
+    if parsed.query:
+        digest = hashlib.sha1(parsed.query.encode()).hexdigest()[:8]
+        stem = PurePosixPath(rel)
+        rel = str(stem.with_name(f"{stem.stem}_{digest}{stem.suffix}"))
+
+    return output_dir / host_dir / rel
+
+
+def page_local_path(url: str, output_dir: Path, lang: str = "html") -> Path:
+    """Local path for a saved page."""
+    ext = ".md" if lang == "md" else ".html"
+    return url_to_local_path(url, output_dir, page_ext=ext)
+
+
+def relative_href(from_path: Path, to_path: Path) -> str:
+    """POSIX-style relative path between two local files."""
+    rel = os.path.relpath(to_path, start=from_path.parent)
+    return PurePosixPath(rel).as_posix()
+
+
+def _safe_name(value: str) -> str:
+    return re.sub(r"[^\w.\-]", "_", value)
+
+
+def _looks_like_page(url: str) -> bool:
+    path = urlparse(url).path
+    ext = PurePosixPath(path).suffix.lower()
+    return ext in PAGE_EXTENSIONS
+
+
+def _guess_extension(url: str) -> str | None:
+    path = urlparse(url).path
+    ext = PurePosixPath(path).suffix.lower()
+    if ext:
+        return ext
+    guessed = mimetypes.guess_extension(mimetypes.guess_type(path)[0] or "application/octet-stream")
+    return guessed
+
+
+def parse_srcset(value: str) -> list[str]:
+    """Extract URLs from a srcset attribute."""
+    urls: list[str] = []
+    for part in value.split(","):
+        piece = part.strip().split()
+        if piece:
+            urls.append(piece[0])
+    return urls
+
+
+def rewrite_css_urls(css: str, base_url: str, mapper) -> str:
+    """Rewrite url(...) references in CSS using mapper(url) -> local path or None."""
+
+    def repl(match: re.Match[str]) -> str:
+        raw = match.group(1).strip("'\"")
+        absolute = normalize_url(raw, base_url)
+        if not absolute:
+            return match.group(0)
+        local = mapper(absolute)
+        if local is None:
+            return match.group(0)
+        return f"url({local})"
+
+    return re.sub(r"url\(([^)]+)\)", repl, css)

web_snapshot_cli-0.1.0.dist-info/METADATA

@@ -0,0 +1,204 @@
+Metadata-Version: 2.4
+Name: web-snapshot-cli
+Version: 0.1.0
+Summary: Take offline snapshots of any website and restore them locally
+Project-URL: Homepage, https://github.com/codingsushi79/Snapshot
+Project-URL: Repository, https://github.com/codingsushi79/Snapshot
+Author: snapshot contributors
+License-Expression: MIT
+License-File: LICENSE
+Keywords: cli,crawler,offline,snapshot,web-archive
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Utilities
+Requires-Python: >=3.10
+Requires-Dist: beautifulsoup4>=4.12
+Requires-Dist: click>=8.1
+Requires-Dist: httpx>=0.27
+Requires-Dist: markdownify>=0.13
+Requires-Dist: rich>=13.7
+Provides-Extra: dev
+Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.8; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# snapshot — offline website snapshots
+
+Take a fast offline copy of any public website, then serve it locally.
+
+## Install
+
+**One-liner (macOS / Linux / WSL):**
+
+```bash
+curl -fsSL https://raw.githubusercontent.com/codingsushi79/Snapshot/main/install.sh | bash
+```
+
+**One-liner (Windows PowerShell):**
+
+```powershell
+irm https://raw.githubusercontent.com/codingsushi79/Snapshot/main/install.ps1 | iex
+```
+
+**Universal fallback** (requires Python 3.10+):
+
+```bash
+python3 -m pip install --user git+https://github.com/codingsushi79/Snapshot.git
+```
+
+**From PyPI** (once published):
+
+```bash
+pip install web-snapshot-cli
+```
+
+Then run:
+
+```bash
+snapshot https://example.com ./mirror
+```
+
+<details>
+<summary>Other install methods</summary>
+
+```bash
+# With pipx (recommended for CLI tools)
+pipx install web-snapshot-cli
+# or from git:
+pipx install git+https://github.com/codingsushi79/Snapshot.git
+
+# From a local clone
+git clone https://github.com/codingsushi79/Snapshot.git
+cd Snapshot
+pip install -e .
+```
+
+</details>
+
+## Usage
+
+### Snapshot a single page
+
+```bash
+snapshot https://example.com ./mirror
+```
+
+### Crawl an entire site
+
+```bash
+snapshot --crawl https://docs.example.com ./docs --max-pages 200 --depth 4
+```
+
+### Save pages as Markdown
+
+```bash
+snapshot https://example.com ./mirror --lang md
+```
+
+### Crawl with filters and politeness
+
+```bash
+snapshot --crawl --include '/docs/*' --exclude '/docs/drafts/*' \
+  --crawl-delay 1 --robots https://docs.example.com ./docs
+```
+
+### Authenticated pages
+
+```bash
+snapshot --cookie session=abc123 --header "Authorization: Bearer TOKEN" \
+  https://app.example.com ./mirror
+```
+
+### Sitemap-based crawl
+
+```bash
+snapshot --crawl --sitemap https://example.com ./mirror --max-pages 500
+```
+
+### Resume an interrupted snapshot
+
+```bash
+snapshot --resume --crawl https://example.com ./mirror
+```
+
+### Dry run (no writes)
+
+```bash
+snapshot --dry-run --verbose --crawl https://example.com ./mirror
+```
+
+### Extra args (positional)
+
+Any `key=value` pairs after the output directory are merged into options:
+
+```bash
+snapshot https://example.com ./mirror crawl=true max-pages=100 lang=html concurrency=32
+```
+
+### Restore locally
+
+```bash
+snapshot -restore ./mirror
+```
+
+This starts a local HTTP server (default `http://127.0.0.1:8080`) and opens your browser.
+
+```bash
+snapshot -restore ./mirror --port 3000 --no-open
+```
+
+## Options
+
+| Flag | Description |
+|------|-------------|
+| `--crawl`, `-c` | Follow same-origin links and download all pages |
+| `--lang`, `-l` | Output format: `html` (default) or `md` |
+| `--max-pages` | Max pages when crawling (default: 50) |
+| `--depth` | Max crawl depth (default: 3) |
+| `--no-assets` | Skip CSS, JS, images, fonts |
+| `--timeout` | HTTP timeout in seconds (default: 15) |
+| `--concurrency` | Parallel downloads (default: 16) |
+| `--same-origin` / `--no-same-origin` | Restrict crawl to same origin (default: on) |
+| `--user-agent` | Custom User-Agent header |
+| `--cookie` | Cookie as `name=value` (repeatable) |
+| `--header` | Extra HTTP header (repeatable) |
+| `--include` | Only fetch URLs matching glob (repeatable) |
+| `--exclude` | Skip URLs matching glob (repeatable) |
+| `--robots` / `--no-robots` | Respect robots.txt (default: on) |
+| `--crawl-delay` | Seconds to wait after each request (default: 0) |
+| `--sitemap` | Seed crawl from sitemap.xml |
+| `--resume` | Skip pages/assets already saved |
+| `--verbose`, `-v` | Detailed log output |
+| `--dry-run` | Fetch without writing files |
+| `-restore DIR` | Serve a saved snapshot from `DIR` |
+| `--port` | Port for restore (default: 8080) |
+| `--host` | Host for restore (default: 127.0.0.1) |
+| `--no-open` | Don't open a browser on restore |
+
+## How it works
+
+1. **snapshot** fetches pages with async HTTP (httpx), rewrites links to local paths, and downloads linked assets in parallel.
+2. A `.snapshot.json` manifest is written to the output folder with metadata for restore.
+3. **snapshot -restore** serves the saved files and maps `/` back to the original root page.
+
+## Output layout
+
+```
+mirror/
+  .snapshot.json
+  example.com/
+    index.html
+    about/
+      index.html
+    _assets/
+      ...
+```
+
+## License
+
+MIT

web_snapshot_cli-0.1.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+snapshot/__init__.py,sha256=8-ApU_fAzbIwD0S_dLk4H7rehrqbrgU4o6Ra_fUEsN0,69
+snapshot/__main__.py,sha256=LXvMYduq7SUnoKWE6DENN8jA_01Aq_6-7Pw7KsLXBh4,69
+snapshot/cli.py,sha256=qczb369TSoM6-bCI1RSpR75QMQT5LUhvMHFdZW6AMks,10398
+snapshot/crawl_policy.py,sha256=aqKoL533VbQwROawqqc9w_mQSFtfUxzYxcs0KJ11iAo,4698
+snapshot/downloader.py,sha256=jNOG6kfbG_8i13qEnlAAEnUpw4BZ1WcWzfNO4b2WGE8,18272
+snapshot/manifest.py,sha256=WFfHJB0urB9n6hb9vFNM5j8QrnAQgOzezQXwzzNEPx4,1759
+snapshot/restore.py,sha256=XIiS4W2xRaIHcdCkixiY14KS2yL_HrLy0WSPS6PvjtM,4193
+snapshot/utils.py,sha256=SgW2CWFFwn9ugIj-5BKQGUySPJIf4pJzlIs2GeO81fU,4352
+web_snapshot_cli-0.1.0.dist-info/METADATA,sha256=1Zeixh_UyLEoik3sFbqskIkwGddJqe0ueX6TgAbVaow,5232
+web_snapshot_cli-0.1.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+web_snapshot_cli-0.1.0.dist-info/entry_points.txt,sha256=Mbqy6jJjTTn5JuLCnmV8G_eJbX5oPKAb7C5qCgnEiXM,47
+web_snapshot_cli-0.1.0.dist-info/licenses/LICENSE,sha256=jkBcwqVvEh626gr4L1xJ5uGcRH0ER7-SwZv9PgW1MiU,1078
+web_snapshot_cli-0.1.0.dist-info/RECORD,,

web_snapshot_cli-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any

web_snapshot_cli-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+snapshot = snapshot.cli:main

web_snapshot_cli-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 snapshot contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.