web-snapshot-cli 0.1.0__py3-none-any.whl

snapshot/downloader.py

@@ -0,0 +1,483 @@
+from __future__ import annotations
+
+import asyncio
+import re
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import httpx
+from bs4 import BeautifulSoup
+from markdownify import markdownify
+
+from snapshot.crawl_policy import (
+    RobotsChecker,
+    discover_sitemap_urls,
+    url_matches_filters,
+)
+from snapshot.manifest import MANIFEST_NAME, SnapshotManifest
+from snapshot.utils import (
+    SRCSET_ATTRS,
+    URL_ATTRS,
+    normalize_url,
+    page_local_path,
+    parse_srcset,
+    relative_href,
+    rewrite_css_urls,
+    same_origin,
+    url_to_local_path,
+)
+
+
+@dataclass
+class SnapshotOptions:
+    crawl: bool = False
+    max_pages: int = 50
+    max_depth: int = 3
+    lang: str = "html"
+    download_assets: bool = True
+    timeout: float = 15.0
+    concurrency: int = 16
+    same_origin_only: bool = True
+    user_agent: str = "web-snapshot-cli/0.1 (+https://github.com/codingsushi79/Snapshot)"
+    cookies: list[str] = field(default_factory=list)
+    headers: list[str] = field(default_factory=list)
+    include_patterns: list[str] = field(default_factory=list)
+    exclude_patterns: list[str] = field(default_factory=list)
+    respect_robots: bool = True
+    crawl_delay: float = 0.0
+    use_sitemap: bool = False
+    resume: bool = False
+    verbose: bool = False
+    dry_run: bool = False
+
+
+@dataclass
+class SnapshotResult:
+    pages_saved: int = 0
+    assets_saved: int = 0
+    pages_skipped: int = 0
+    assets_skipped: int = 0
+    errors: list[str] = field(default_factory=list)
+
+
+class SnapshotEngine:
+    def __init__(self, root_url: str, output_dir: Path, options: SnapshotOptions):
+        self.root_url = normalize_url(root_url) or root_url
+        self.output_dir = output_dir.resolve()
+        self.options = options
+        self._seen_pages: set[str] = set()
+        self._queued_pages: set[str] = set()
+        self._seen_assets: set[str] = set()
+        self._url_to_path: dict[str, Path] = {}
+        self._errors: list[str] = []
+        self._sem: asyncio.Semaphore | None = None
+        self._robots = RobotsChecker(options.user_agent, options.respect_robots)
+        self._pages_skipped = 0
+        self._assets_skipped = 0
+        self._log_fn: Callable[[str], None] | None = None
+
+    async def run(
+        self,
+        progress: Callable[[str], None] | None = None,
+        log: Callable[[str], None] | None = None,
+    ) -> SnapshotResult:
+        self._log_fn = log
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+        self._sem = asyncio.Semaphore(self.options.concurrency)
+        self._load_resume_state()
+
+        client_headers = self._build_headers()
+        client_cookies = self._build_cookies()
+
+        limits = httpx.Limits(max_connections=self.options.concurrency * 2)
+        async with httpx.AsyncClient(
+            follow_redirects=True,
+            timeout=self.options.timeout,
+            headers=client_headers,
+            cookies=client_cookies,
+            limits=limits,
+        ) as client:
+            self._client = client
+            if self.options.crawl:
+                await self._crawl(progress)
+            else:
+                await self._snapshot_page(self.root_url, depth=0, progress=progress)
+
+        if not self.options.dry_run:
+            manifest = SnapshotManifest(
+                root_url=self.root_url,
+                output_dir=str(self.output_dir),
+                pages=sorted(self._seen_pages),
+                assets=sorted(self._seen_assets),
+                options=self._manifest_options(),
+            )
+            manifest.save(self.output_dir)
+
+        return SnapshotResult(
+            pages_saved=len(self._seen_pages),
+            assets_saved=len(self._seen_assets),
+            pages_skipped=self._pages_skipped,
+            assets_skipped=self._assets_skipped,
+            errors=self._errors,
+        )
+
+    def _manifest_options(self) -> dict:
+        return {
+            "crawl": self.options.crawl,
+            "max_pages": self.options.max_pages,
+            "max_depth": self.options.max_depth,
+            "lang": self.options.lang,
+            "download_assets": self.options.download_assets,
+            "same_origin_only": self.options.same_origin_only,
+            "respect_robots": self.options.respect_robots,
+            "crawl_delay": self.options.crawl_delay,
+            "use_sitemap": self.options.use_sitemap,
+        }
+
+    def _build_headers(self) -> dict[str, str]:
+        headers = {"User-Agent": self.options.user_agent}
+        for item in self.options.headers:
+            if ":" not in item:
+                continue
+            name, value = item.split(":", 1)
+            headers[name.strip()] = value.strip()
+        return headers
+
+    def _build_cookies(self) -> dict[str, str]:
+        cookies: dict[str, str] = {}
+        for item in self.options.cookies:
+            if "=" not in item:
+                continue
+            name, value = item.split("=", 1)
+            cookies[name.strip()] = value.strip()
+        return cookies
+
+    def _load_resume_state(self) -> None:
+        if not self.options.resume:
+            return
+        manifest_path = self.output_dir / MANIFEST_NAME
+        if not manifest_path.exists():
+            self._log("resume: no existing manifest, starting fresh")
+            return
+
+        manifest = SnapshotManifest.load(self.output_dir)
+        page_ext = ".md" if self.options.lang == "md" else ".html"
+        for page_url in manifest.pages:
+            local_path = page_local_path(page_url, self.output_dir, lang=self.options.lang)
+            self._seen_pages.add(page_url)
+            self._url_to_path[page_url] = local_path
+        for asset_url in manifest.assets:
+            local_path = url_to_local_path(asset_url, self.output_dir, page_ext=page_ext)
+            self._seen_assets.add(asset_url)
+            self._url_to_path[asset_url] = local_path
+        self._log(
+            f"resume: loaded {len(manifest.pages)} pages, "
+            f"{len(manifest.assets)} assets from manifest"
+        )
+
+    def _log(self, message: str) -> None:
+        if self._log_fn:
+            self._log_fn(message)
+
+    def _url_allowed(self, url: str) -> bool:
+        if self.options.same_origin_only and not same_origin(self.root_url, url):
+            return False
+        return url_matches_filters(
+            url, self.options.include_patterns, self.options.exclude_patterns
+        )
+
+    async def _crawl(self, progress: Callable[[str], None] | None) -> None:
+        queue: asyncio.Queue[tuple[str, int]] = asyncio.Queue()
+        seed_urls = [self.root_url]
+
+        if self.options.use_sitemap:
+            if progress:
+                progress("fetching sitemap…")
+            sitemap_urls = await discover_sitemap_urls(self._client, self.root_url, self._robots)
+            self._log(f"sitemap: discovered {len(sitemap_urls)} URLs")
+            seed_urls.extend(sitemap_urls)
+
+        for url in seed_urls:
+            normalized = normalize_url(url)
+            if not normalized or not self._url_allowed(normalized):
+                continue
+            if normalized in self._queued_pages:
+                continue
+            self._queued_pages.add(normalized)
+            await queue.put((normalized, 0))
+
+        workers = [
+            asyncio.create_task(self._crawl_worker(queue, progress))
+            for _ in range(min(self.options.concurrency, 8))
+        ]
+        await queue.join()
+        for worker in workers:
+            worker.cancel()
+        await asyncio.gather(*workers, return_exceptions=True)
+
+    async def _crawl_worker(
+        self,
+        queue: asyncio.Queue[tuple[str, int]],
+        progress: Callable[[str], None] | None,
+    ) -> None:
+        while True:
+            url, depth = await queue.get()
+            try:
+                if len(self._seen_pages) >= self.options.max_pages:
+                    continue
+                if depth > self.options.max_depth:
+                    continue
+                if not self._url_allowed(url):
+                    self._log(f"skip (filter): {url}")
+                    continue
+                if not await self._robots.can_fetch(self._client, url):
+                    self._log(f"skip (robots.txt): {url}")
+                    continue
+
+                if url in self._seen_pages and self._page_exists(url):
+                    self._pages_skipped += 1
+                    self._log(f"skip (resume): {url}")
+                    links = self._links_from_saved_page(url)
+                else:
+                    links = await self._snapshot_page(url, depth=depth, progress=progress)
+
+                if depth < self.options.max_depth and links:
+                    for link in links:
+                        if len(self._seen_pages) >= self.options.max_pages:
+                            break
+                        if link in self._queued_pages:
+                            continue
+                        if not self._url_allowed(link):
+                            continue
+                        self._queued_pages.add(link)
+                        await queue.put((link, depth + 1))
+            finally:
+                queue.task_done()
+
+    def _page_exists(self, url: str) -> bool:
+        local_path = self._url_to_path.get(url)
+        if local_path is None:
+            local_path = page_local_path(url, self.output_dir, lang=self.options.lang)
+        return local_path.exists()
+
+    def _links_from_saved_page(self, url: str) -> list[str]:
+        local_path = self._url_to_path.get(url) or page_local_path(
+            url, self.output_dir, lang=self.options.lang
+        )
+        if not local_path.exists():
+            return []
+        try:
+            html = local_path.read_text(encoding="utf-8")
+        except OSError:
+            return []
+        soup = BeautifulSoup(html, "html.parser")
+        return self._collect_page_links(soup, url)
+
+    async def _snapshot_page(
+        self,
+        url: str,
+        depth: int,
+        progress: Callable[[str], None] | None,
+    ) -> list[str]:
+        if url in self._seen_pages and self._page_exists(url):
+            self._pages_skipped += 1
+            return self._links_from_saved_page(url)
+
+        if not self._url_allowed(url):
+            return []
+        if not await self._robots.can_fetch(self._client, url):
+            self._log(f"skip (robots.txt): {url}")
+            return []
+
+        if progress:
+            progress(f"page {url}")
+        self._log(f"fetch: {url}")
+
+        try:
+            response = await self._fetch(url)
+        except Exception as exc:  # noqa: BLE001
+            self._errors.append(f"{url}: {exc}")
+            return []
+
+        self._seen_pages.add(url)
+
+        content_type = response.headers.get("content-type", "")
+        local_path = page_local_path(url, self.output_dir, lang=self.options.lang)
+        self._url_to_path[url] = local_path
+
+        if "text/html" not in content_type and not url.endswith((".html", ".htm")):
+            if not self.options.dry_run:
+                local_path.parent.mkdir(parents=True, exist_ok=True)
+                local_path.write_bytes(response.content)
+            else:
+                self._log(f"dry-run: would save {url} → {local_path}")
+            return []
+
+        soup = BeautifulSoup(response.text, "html.parser")
+        discovered = self._collect_page_links(soup, url)
+
+        asset_urls = self._collect_asset_urls(soup, url)
+        if self.options.download_assets:
+            await asyncio.gather(*(self._download_asset(u) for u in asset_urls))
+
+        if not self.options.dry_run:
+            local_path.parent.mkdir(parents=True, exist_ok=True)
+            self._rewrite_html(soup, url, local_path)
+            body = self._render_page(soup)
+            local_path.write_text(body, encoding="utf-8")
+        else:
+            self._log(f"dry-run: would save page {url} → {local_path}")
+
+        return discovered
+
+    def _collect_asset_urls(self, soup: BeautifulSoup, page_url: str) -> set[str]:
+        urls: set[str] = set()
+        for tag in soup.find_all(True):
+            for attr in URL_ATTRS:
+                value = tag.get(attr)
+                if value:
+                    absolute = normalize_url(value, page_url)
+                    if absolute and not self._is_page_url(absolute):
+                        urls.add(absolute)
+            for attr in SRCSET_ATTRS:
+                value = tag.get(attr)
+                if value:
+                    for piece in parse_srcset(value):
+                        absolute = normalize_url(piece, page_url)
+                        if absolute and not self._is_page_url(absolute):
+                            urls.add(absolute)
+        for style in soup.find_all("style"):
+            if style.string:
+                for match in re.findall(r"url\(([^)]+)\)", style.string):
+                    absolute = normalize_url(match.strip("'\""), page_url)
+                    if absolute and not self._is_page_url(absolute):
+                        urls.add(absolute)
+        return urls
+
+    def _collect_page_links(self, soup: BeautifulSoup, page_url: str) -> list[str]:
+        links: list[str] = []
+        for tag in soup.find_all("a", href=True):
+            absolute = normalize_url(tag["href"], page_url)
+            if absolute and self._is_page_url(absolute):
+                links.append(absolute)
+        return links
+
+    def _is_page_url(self, url: str) -> bool:
+        from snapshot.utils import _looks_like_page
+
+        return _looks_like_page(url)
+
+    async def _download_asset(self, url: str) -> None:
+        if url in self._seen_assets:
+            return
+        self._seen_assets.add(url)
+        local_path = url_to_local_path(url, self.output_dir)
+        self._url_to_path[url] = local_path
+        if local_path.exists():
+            self._assets_skipped += 1
+            self._log(f"skip (exists): {url}")
+            return
+
+        self._log(f"asset: {url}")
+        try:
+            response = await self._fetch(url)
+            if self.options.dry_run:
+                self._log(f"dry-run: would save asset {url} → {local_path}")
+                return
+            local_path.parent.mkdir(parents=True, exist_ok=True)
+            if url.endswith(".css") or "text/css" in response.headers.get("content-type", ""):
+                css = response.text
+                css = rewrite_css_urls(
+                    css,
+                    url,
+                    lambda asset_url: self._css_mapper(asset_url, local_path),
+                )
+                local_path.write_text(css, encoding="utf-8")
+                nested = self._extract_css_urls(css, url)
+                await asyncio.gather(*(self._download_asset(u) for u in nested))
+            else:
+                local_path.write_bytes(response.content)
+        except Exception as exc:  # noqa: BLE001
+            self._errors.append(f"{url}: {exc}")
+
+    def _css_mapper(self, asset_url: str, css_path: Path) -> str | None:
+        target = self._url_to_path.get(asset_url)
+        if target is None:
+            target = url_to_local_path(asset_url, self.output_dir)
+            self._url_to_path[asset_url] = target
+        return relative_href(css_path, target)
+
+    def _extract_css_urls(self, css: str, base_url: str) -> set[str]:
+        urls: set[str] = set()
+        for match in re.findall(r"url\(([^)]+)\)", css):
+            absolute = normalize_url(match.strip("'\""), base_url)
+            if absolute:
+                urls.add(absolute)
+        return urls
+
+    def _rewrite_html(self, soup: BeautifulSoup, page_url: str, page_path: Path) -> None:
+        for tag in soup.find_all(True):
+            for attr in URL_ATTRS:
+                value = tag.get(attr)
+                if not value:
+                    continue
+                absolute = normalize_url(value, page_url)
+                if not absolute:
+                    continue
+                if absolute not in self._url_to_path and not same_origin(self.root_url, absolute):
+                    continue
+                target = self._url_to_path.get(absolute)
+                if target is None:
+                    target = (
+                        page_local_path(absolute, self.output_dir, lang=self.options.lang)
+                        if self._is_page_url(absolute)
+                        else url_to_local_path(absolute, self.output_dir)
+                    )
+                tag[attr] = relative_href(page_path, target)
+            for attr in SRCSET_ATTRS:
+                value = tag.get(attr)
+                if not value:
+                    continue
+                parts = []
+                for piece in value.split(","):
+                    chunk = piece.strip().split()
+                    if not chunk:
+                        continue
+                    absolute = normalize_url(chunk[0], page_url)
+                    if absolute and absolute in self._url_to_path:
+                        local = relative_href(page_path, self._url_to_path[absolute])
+                        rest = " ".join(chunk[1:])
+                        parts.append(f"{local} {rest}".strip())
+                    else:
+                        parts.append(piece.strip())
+                if parts:
+                    tag[attr] = ", ".join(parts)
+
+        base_tag = soup.find("base")
+        if base_tag:
+            base_tag.decompose()
+
+    def _render_page(self, soup: BeautifulSoup) -> str:
+        if self.options.lang == "md":
+            return self._html_to_markdown(soup)
+        return str(soup)
+
+    def _html_to_markdown(self, soup: BeautifulSoup) -> str:
+        title = soup.title.get_text(strip=True) if soup.title else "Untitled"
+        body = soup.body or soup
+        markdown = markdownify(
+            str(body),
+            heading_style="ATX",
+            bullets="-",
+            strip=["script", "style", "noscript"],
+        ).strip()
+        return f"# {title}\n\n{markdown}\n"
+
+    async def _fetch(self, url: str) -> httpx.Response:
+        async with self._sem:
+            response = await self._client.get(url)
+            response.raise_for_status()
+            if self.options.crawl_delay > 0:
+                await asyncio.sleep(self.options.crawl_delay)
+            return response

snapshot/manifest.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+import json
+from dataclasses import asdict, dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+MANIFEST_NAME = ".snapshot.json"
+
+
+@dataclass
+class SnapshotManifest:
+    version: str = "1"
+    created_at: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat())
+    root_url: str = ""
+    output_dir: str = ""
+    pages: list[str] = field(default_factory=list)
+    assets: list[str] = field(default_factory=list)
+    options: dict[str, Any] = field(default_factory=dict)
+
+    def save(self, output_dir: Path) -> Path:
+        path = output_dir / MANIFEST_NAME
+        path.write_text(json.dumps(asdict(self), indent=2), encoding="utf-8")
+        return path
+
+    @classmethod
+    def load(cls, snapshot_dir: Path) -> SnapshotManifest:
+        path = snapshot_dir / MANIFEST_NAME
+        if not path.exists():
+            raise FileNotFoundError(
+                f"No snapshot manifest found at {path}. "
+                "Make sure this directory was created by snapshot."
+            )
+        data = json.loads(path.read_text(encoding="utf-8"))
+        return cls(**data)
+
+    @classmethod
+    def find_root(cls, snapshot_dir: Path) -> Path:
+        """Return the directory that contains the manifest (may be nested)."""
+        candidate = snapshot_dir.resolve()
+        if (candidate / MANIFEST_NAME).exists():
+            return candidate
+        for child in candidate.iterdir():
+            if child.is_dir() and (child / MANIFEST_NAME).exists():
+                return child
+        raise FileNotFoundError(
+            f"No {MANIFEST_NAME} found in {snapshot_dir}. "
+            "Pass the folder that contains your snapshot files."
+        )

snapshot/restore.py

@@ -0,0 +1,134 @@
+from __future__ import annotations
+
+import functools
+import http.server
+import mimetypes
+import socketserver
+import webbrowser
+from pathlib import Path
+from urllib.parse import urlparse
+
+from snapshot.manifest import SnapshotManifest
+from snapshot.utils import page_local_path
+
+
+def serve_snapshot(
+    snapshot_dir: Path, host: str = "127.0.0.1", port: int = 8080, open_browser: bool = True
+) -> None:
+    """Serve a snapshot directory over HTTP."""
+    root = SnapshotManifest.find_root(snapshot_dir)
+    manifest = SnapshotManifest.load(root)
+    content_root = _content_root(root, manifest)
+
+    handler = functools.partial(
+        _SnapshotHandler,
+        directory=str(content_root),
+        manifest=manifest,
+    )
+
+    with _FindServer(host, port, handler) as httpd:
+        actual_port = httpd.server_address[1]
+        url = f"http://{host}:{actual_port}/"
+        print(f"snapshot restore: serving {content_root}")
+        print(f"  root URL : {manifest.root_url}")
+        print(f"  local URL: {url}")
+        print("  Press Ctrl+C to stop.")
+        if open_browser:
+            webbrowser.open(url)
+        try:
+            httpd.serve_forever()
+        except KeyboardInterrupt:
+            print("\nStopped.")
+
+
+def _content_root(root: Path, manifest: SnapshotManifest) -> Path:
+    """Pick the host directory inside the snapshot output."""
+    parsed = urlparse(manifest.root_url)
+    host_name = parsed.netloc.replace(":", "_")
+    for candidate in root.iterdir():
+        if candidate.is_dir() and host_name in candidate.name:
+            return candidate
+    children = [p for p in root.iterdir() if p.is_dir() and p.name != "__pycache__"]
+    if len(children) == 1:
+        return children[0]
+    return root
+
+
+class _SnapshotHandler(http.server.SimpleHTTPRequestHandler):
+    manifest: SnapshotManifest
+
+    def __init__(self, *args, directory: str, manifest: SnapshotManifest, **kwargs):
+        self.manifest = manifest
+        super().__init__(*args, directory=directory, **kwargs)
+
+    def do_GET(self) -> None:  # noqa: N802
+        path = self.path.split("?", 1)[0]
+        resolved = self._resolve_path(path)
+        if resolved is not None:
+            self.path = "/" + resolved
+        return super().do_GET()
+
+    def _resolve_path(self, path: str) -> str | None:
+        root = Path(self.directory)
+        lang = self.manifest.options.get("lang", "html")
+        ext = ".md" if lang == "md" else ".html"
+
+        if path in {"/", ""}:
+            local = page_local_path(self.manifest.root_url, root, lang=lang)
+            if local.exists():
+                return local.relative_to(root).as_posix()
+            return None
+
+        clean = path.strip("/")
+        if not clean:
+            return None
+
+        direct = root / clean
+        if direct.is_file():
+            return clean
+
+        index = direct / f"index{ext}"
+        if index.is_file():
+            return index.relative_to(root).as_posix()
+
+        with_ext = root / f"{clean}{ext}"
+        if with_ext.is_file():
+            return with_ext.relative_to(root).as_posix()
+
+        return None
+
+    def log_message(self, format: str, *args) -> None:  # noqa: A003
+        return
+
+
+class _FindServer:
+    def __init__(self, host: str, port: int, handler):
+        self.host = host
+        self.port = port
+        self.handler = handler
+        self.httpd = None
+
+    def __enter__(self):
+        for attempt in range(20):
+            try:
+                self.httpd = socketserver.TCPServer(
+                    (self.host, self.port + attempt),
+                    self.handler,
+                )
+                self.httpd.allow_reuse_address = True
+                return self.httpd
+            except OSError:
+                continue
+        raise OSError(f"Could not bind to {self.host}:{self.port}+")
+
+    def __exit__(self, exc_type, exc, tb):
+        if self.httpd:
+            self.httpd.server_close()
+
+
+# Ensure common web types are registered.
+mimetypes.add_type("application/javascript", ".js")
+mimetypes.add_type("application/javascript", ".mjs")
+mimetypes.add_type("text/css", ".css")
+mimetypes.add_type("image/webp", ".webp")
+mimetypes.add_type("font/woff2", ".woff2")