web-snapshot-cli 0.1.0__tar.gz

web_snapshot_cli-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,35 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install package and dev dependencies
+        run: pip install -e ".[dev]"
+
+      - name: Ruff
+        run: ruff check src tests
+
+      - name: Ruff format
+        run: ruff format --check src tests
+
+      - name: Pytest
+        run: pytest

web_snapshot_cli-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,64 @@
+# PyPI project: web-snapshot-cli
+# Trusted publisher: owner codingsushi79, repo Snapshot, workflow publish.yml, env pypi
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install package and dev dependencies
+        run: pip install -e ".[dev]"
+
+      - name: Ruff
+        run: ruff check src tests
+
+      - name: Pytest
+        run: pytest
+
+  build:
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Build sdist and wheel
+        run: |
+          pip install build
+          python -m build
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - uses: pypa/gh-action-pypi-publish@release/v1

web_snapshot_cli-0.1.0/.gitignore

@@ -0,0 +1,15 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+.eggs/
+dist/
+build/
+.venv/
+venv/
+.env
+.DS_Store
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+*.log
+/tmp/

web_snapshot_cli-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 snapshot contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

web_snapshot_cli-0.1.0/PKG-INFO

@@ -0,0 +1,204 @@
+Metadata-Version: 2.4
+Name: web-snapshot-cli
+Version: 0.1.0
+Summary: Take offline snapshots of any website and restore them locally
+Project-URL: Homepage, https://github.com/codingsushi79/Snapshot
+Project-URL: Repository, https://github.com/codingsushi79/Snapshot
+Author: snapshot contributors
+License-Expression: MIT
+License-File: LICENSE
+Keywords: cli,crawler,offline,snapshot,web-archive
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Utilities
+Requires-Python: >=3.10
+Requires-Dist: beautifulsoup4>=4.12
+Requires-Dist: click>=8.1
+Requires-Dist: httpx>=0.27
+Requires-Dist: markdownify>=0.13
+Requires-Dist: rich>=13.7
+Provides-Extra: dev
+Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.8; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# snapshot — offline website snapshots
+
+Take a fast offline copy of any public website, then serve it locally.
+
+## Install
+
+**One-liner (macOS / Linux / WSL):**
+
+```bash
+curl -fsSL https://raw.githubusercontent.com/codingsushi79/Snapshot/main/install.sh | bash
+```
+
+**One-liner (Windows PowerShell):**
+
+```powershell
+irm https://raw.githubusercontent.com/codingsushi79/Snapshot/main/install.ps1 | iex
+```
+
+**Universal fallback** (requires Python 3.10+):
+
+```bash
+python3 -m pip install --user git+https://github.com/codingsushi79/Snapshot.git
+```
+
+**From PyPI** (once published):
+
+```bash
+pip install web-snapshot-cli
+```
+
+Then run:
+
+```bash
+snapshot https://example.com ./mirror
+```
+
+<details>
+<summary>Other install methods</summary>
+
+```bash
+# With pipx (recommended for CLI tools)
+pipx install web-snapshot-cli
+# or from git:
+pipx install git+https://github.com/codingsushi79/Snapshot.git
+
+# From a local clone
+git clone https://github.com/codingsushi79/Snapshot.git
+cd Snapshot
+pip install -e .
+```
+
+</details>
+
+## Usage
+
+### Snapshot a single page
+
+```bash
+snapshot https://example.com ./mirror
+```
+
+### Crawl an entire site
+
+```bash
+snapshot --crawl https://docs.example.com ./docs --max-pages 200 --depth 4
+```
+
+### Save pages as Markdown
+
+```bash
+snapshot https://example.com ./mirror --lang md
+```
+
+### Crawl with filters and politeness
+
+```bash
+snapshot --crawl --include '/docs/*' --exclude '/docs/drafts/*' \
+  --crawl-delay 1 --robots https://docs.example.com ./docs
+```
+
+### Authenticated pages
+
+```bash
+snapshot --cookie session=abc123 --header "Authorization: Bearer TOKEN" \
+  https://app.example.com ./mirror
+```
+
+### Sitemap-based crawl
+
+```bash
+snapshot --crawl --sitemap https://example.com ./mirror --max-pages 500
+```
+
+### Resume an interrupted snapshot
+
+```bash
+snapshot --resume --crawl https://example.com ./mirror
+```
+
+### Dry run (no writes)
+
+```bash
+snapshot --dry-run --verbose --crawl https://example.com ./mirror
+```
+
+### Extra args (positional)
+
+Any `key=value` pairs after the output directory are merged into options:
+
+```bash
+snapshot https://example.com ./mirror crawl=true max-pages=100 lang=html concurrency=32
+```
+
+### Restore locally
+
+```bash
+snapshot -restore ./mirror
+```
+
+This starts a local HTTP server (default `http://127.0.0.1:8080`) and opens your browser.
+
+```bash
+snapshot -restore ./mirror --port 3000 --no-open
+```
+
+## Options
+
+| Flag | Description |
+|------|-------------|
+| `--crawl`, `-c` | Follow same-origin links and download all pages |
+| `--lang`, `-l` | Output format: `html` (default) or `md` |
+| `--max-pages` | Max pages when crawling (default: 50) |
+| `--depth` | Max crawl depth (default: 3) |
+| `--no-assets` | Skip CSS, JS, images, fonts |
+| `--timeout` | HTTP timeout in seconds (default: 15) |
+| `--concurrency` | Parallel downloads (default: 16) |
+| `--same-origin` / `--no-same-origin` | Restrict crawl to same origin (default: on) |
+| `--user-agent` | Custom User-Agent header |
+| `--cookie` | Cookie as `name=value` (repeatable) |
+| `--header` | Extra HTTP header (repeatable) |
+| `--include` | Only fetch URLs matching glob (repeatable) |
+| `--exclude` | Skip URLs matching glob (repeatable) |
+| `--robots` / `--no-robots` | Respect robots.txt (default: on) |
+| `--crawl-delay` | Seconds to wait after each request (default: 0) |
+| `--sitemap` | Seed crawl from sitemap.xml |
+| `--resume` | Skip pages/assets already saved |
+| `--verbose`, `-v` | Detailed log output |
+| `--dry-run` | Fetch without writing files |
+| `-restore DIR` | Serve a saved snapshot from `DIR` |
+| `--port` | Port for restore (default: 8080) |
+| `--host` | Host for restore (default: 127.0.0.1) |
+| `--no-open` | Don't open a browser on restore |
+
+## How it works
+
+1. **snapshot** fetches pages with async HTTP (httpx), rewrites links to local paths, and downloads linked assets in parallel.
+2. A `.snapshot.json` manifest is written to the output folder with metadata for restore.
+3. **snapshot -restore** serves the saved files and maps `/` back to the original root page.
+
+## Output layout
+
+```
+mirror/
+  .snapshot.json
+  example.com/
+    index.html
+    about/
+      index.html
+    _assets/
+      ...
+```
+
+## License
+
+MIT

web_snapshot_cli-0.1.0/README.md

@@ -0,0 +1,175 @@
+# snapshot — offline website snapshots
+
+Take a fast offline copy of any public website, then serve it locally.
+
+## Install
+
+**One-liner (macOS / Linux / WSL):**
+
+```bash
+curl -fsSL https://raw.githubusercontent.com/codingsushi79/Snapshot/main/install.sh | bash
+```
+
+**One-liner (Windows PowerShell):**
+
+```powershell
+irm https://raw.githubusercontent.com/codingsushi79/Snapshot/main/install.ps1 | iex
+```
+
+**Universal fallback** (requires Python 3.10+):
+
+```bash
+python3 -m pip install --user git+https://github.com/codingsushi79/Snapshot.git
+```
+
+**From PyPI** (once published):
+
+```bash
+pip install web-snapshot-cli
+```
+
+Then run:
+
+```bash
+snapshot https://example.com ./mirror
+```
+
+<details>
+<summary>Other install methods</summary>
+
+```bash
+# With pipx (recommended for CLI tools)
+pipx install web-snapshot-cli
+# or from git:
+pipx install git+https://github.com/codingsushi79/Snapshot.git
+
+# From a local clone
+git clone https://github.com/codingsushi79/Snapshot.git
+cd Snapshot
+pip install -e .
+```
+
+</details>
+
+## Usage
+
+### Snapshot a single page
+
+```bash
+snapshot https://example.com ./mirror
+```
+
+### Crawl an entire site
+
+```bash
+snapshot --crawl https://docs.example.com ./docs --max-pages 200 --depth 4
+```
+
+### Save pages as Markdown
+
+```bash
+snapshot https://example.com ./mirror --lang md
+```
+
+### Crawl with filters and politeness
+
+```bash
+snapshot --crawl --include '/docs/*' --exclude '/docs/drafts/*' \
+  --crawl-delay 1 --robots https://docs.example.com ./docs
+```
+
+### Authenticated pages
+
+```bash
+snapshot --cookie session=abc123 --header "Authorization: Bearer TOKEN" \
+  https://app.example.com ./mirror
+```
+
+### Sitemap-based crawl
+
+```bash
+snapshot --crawl --sitemap https://example.com ./mirror --max-pages 500
+```
+
+### Resume an interrupted snapshot
+
+```bash
+snapshot --resume --crawl https://example.com ./mirror
+```
+
+### Dry run (no writes)
+
+```bash
+snapshot --dry-run --verbose --crawl https://example.com ./mirror
+```
+
+### Extra args (positional)
+
+Any `key=value` pairs after the output directory are merged into options:
+
+```bash
+snapshot https://example.com ./mirror crawl=true max-pages=100 lang=html concurrency=32
+```
+
+### Restore locally
+
+```bash
+snapshot -restore ./mirror
+```
+
+This starts a local HTTP server (default `http://127.0.0.1:8080`) and opens your browser.
+
+```bash
+snapshot -restore ./mirror --port 3000 --no-open
+```
+
+## Options
+
+| Flag | Description |
+|------|-------------|
+| `--crawl`, `-c` | Follow same-origin links and download all pages |
+| `--lang`, `-l` | Output format: `html` (default) or `md` |
+| `--max-pages` | Max pages when crawling (default: 50) |
+| `--depth` | Max crawl depth (default: 3) |
+| `--no-assets` | Skip CSS, JS, images, fonts |
+| `--timeout` | HTTP timeout in seconds (default: 15) |
+| `--concurrency` | Parallel downloads (default: 16) |
+| `--same-origin` / `--no-same-origin` | Restrict crawl to same origin (default: on) |
+| `--user-agent` | Custom User-Agent header |
+| `--cookie` | Cookie as `name=value` (repeatable) |
+| `--header` | Extra HTTP header (repeatable) |
+| `--include` | Only fetch URLs matching glob (repeatable) |
+| `--exclude` | Skip URLs matching glob (repeatable) |
+| `--robots` / `--no-robots` | Respect robots.txt (default: on) |
+| `--crawl-delay` | Seconds to wait after each request (default: 0) |
+| `--sitemap` | Seed crawl from sitemap.xml |
+| `--resume` | Skip pages/assets already saved |
+| `--verbose`, `-v` | Detailed log output |
+| `--dry-run` | Fetch without writing files |
+| `-restore DIR` | Serve a saved snapshot from `DIR` |
+| `--port` | Port for restore (default: 8080) |
+| `--host` | Host for restore (default: 127.0.0.1) |
+| `--no-open` | Don't open a browser on restore |
+
+## How it works
+
+1. **snapshot** fetches pages with async HTTP (httpx), rewrites links to local paths, and downloads linked assets in parallel.
+2. A `.snapshot.json` manifest is written to the output folder with metadata for restore.
+3. **snapshot -restore** serves the saved files and maps `/` back to the original root page.
+
+## Output layout
+
+```
+mirror/
+  .snapshot.json
+  example.com/
+    index.html
+    about/
+      index.html
+    _assets/
+      ...
+```
+
+## License
+
+MIT

web_snapshot_cli-0.1.0/install.ps1

@@ -0,0 +1,67 @@
+#Requires -Version 5.1
+$ErrorActionPreference = "Stop"
+
+$Repo = if ($env:SNAPSHOT_REPO) { $env:SNAPSHOT_REPO } else { "https://github.com/codingsushi79/Snapshot.git" }
+$Ref = if ($env:SNAPSHOT_REF) { $env:SNAPSHOT_REF } else { "main" }
+$InstallSpec = "git+$Repo@$Ref"
+
+function Write-Info($msg) { Write-Host "→ $msg" -ForegroundColor Cyan }
+function Write-Warn($msg) { Write-Host "! $msg" -ForegroundColor Yellow }
+function Write-Err($msg)  { Write-Host "✗ $msg" -ForegroundColor Red }
+
+function Find-Python {
+    foreach ($cmd in @("python", "python3", "py")) {
+        if (Get-Command $cmd -ErrorAction SilentlyContinue) {
+            $ok = & $cmd -c "import sys; raise SystemExit(0 if sys.version_info >= (3,10) else 1)" 2>$null
+            if ($LASTEXITCODE -eq 0) { return $cmd }
+        }
+    }
+    return $null
+}
+
+function Ensure-Pip($py) {
+    & $py -m pip --version 2>$null | Out-Null
+    if ($LASTEXITCODE -ne 0) {
+        Write-Info "Bootstrapping pip…"
+        & $py -m ensurepip --upgrade 2>$null | Out-Null
+    }
+    & $py -m pip --version 2>$null | Out-Null
+    return ($LASTEXITCODE -eq 0)
+}
+
+Write-Info "Installing snapshot from $InstallSpec"
+
+$py = Find-Python
+if (-not $py) {
+    Write-Err "Python 3.10+ is required."
+    Write-Err "Install from https://www.python.org/downloads/ and re-run this script."
+    exit 1
+}
+
+if (-not (Ensure-Pip $py)) {
+    Write-Err "pip is required but could not be installed."
+    exit 1
+}
+
+if (Get-Command pipx -ErrorAction SilentlyContinue) {
+    Write-Info "Installing snapshot with pipx…"
+    pipx install --force $InstallSpec
+} else {
+    Write-Info "Installing snapshot with pip…"
+    & $py -m pip install --user --upgrade $InstallSpec
+}
+
+$localBin = Join-Path $env:USERPROFILE ".local\\bin"
+$scriptsDir = Join-Path $env:APPDATA "Python\\Python312\\Scripts"
+$paths = @($localBin, $scriptsDir) | Where-Object { Test-Path $_ }
+if ($paths.Count -gt 0) {
+    $env:PATH = ($paths -join ";") + ";" + $env:PATH
+}
+
+if (Get-Command snapshot -ErrorAction SilentlyContinue) {
+    Write-Info "Installed snapshot."
+    Write-Info "Run: snapshot https://example.com ./mirror"
+} else {
+    Write-Warn "snapshot installed, but not on PATH."
+    Write-Warn "Restart your terminal or add Python Scripts to PATH."
+}

web_snapshot_cli-0.1.0/install.sh

@@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+REPO="${SNAPSHOT_REPO:-https://github.com/codingsushi79/Snapshot.git}"
+REF="${SNAPSHOT_REF:-main}"
+INSTALL_SPEC="git+${REPO}@${REF}"
+
+info() { printf '\033[1;34m→\033[0m %s\n' "$*"; }
+warn() { printf '\033[1;33m!\033[0m %s\n' "$*"; }
+err()  { printf '\033[1;31m✗\033[0m %s\n' "$*" >&2; }
+
+need_cmd() {
+  command -v "$1" >/dev/null 2>&1
+}
+
+find_python() {
+  for cmd in python3 python py; do
+    if need_cmd "$cmd"; then
+      if "$cmd" -c 'import sys; raise SystemExit(0 if sys.version_info >= (3, 10) else 1)' 2>/dev/null; then
+        echo "$cmd"
+        return 0
+      fi
+    fi
+  done
+  return 1
+}
+
+ensure_pip() {
+  local py="$1"
+  if "$py" -m pip --version >/dev/null 2>&1; then
+    return 0
+  fi
+  info "Bootstrapping pip…"
+  "$py" -m ensurepip --upgrade >/dev/null 2>&1 || true
+  "$py" -m pip --version >/dev/null 2>&1
+}
+
+pip_user_flag() {
+  if [[ -n "${VIRTUAL_ENV:-}" ]]; then
+    echo ""
+  else
+    echo "--user"
+  fi
+}
+
+install_with_pipx() {
+  local py="$1"
+  info "Installing snapshot with pipx…"
+  if ! need_cmd pipx; then
+    local user_flag
+    user_flag="$(pip_user_flag)"
+    # shellcheck disable=SC2086
+    "$py" -m pip install $user_flag pipx
+    export PATH="${HOME}/.local/bin:${PATH}"
+    if need_cmd pipx; then
+      pipx ensurepath >/dev/null 2>&1 || true
+    fi
+  fi
+  if need_cmd pipx; then
+    pipx install --force "$INSTALL_SPEC"
+    return 0
+  fi
+  return 1
+}
+
+install_with_pip() {
+  local py="$1"
+  local user_flag
+  user_flag="$(pip_user_flag)"
+  info "Installing snapshot with pip…"
+  # shellcheck disable=SC2086
+  "$py" -m pip install $user_flag --upgrade "$INSTALL_SPEC"
+}
+
+verify_install() {
+  export PATH="${HOME}/.local/bin:${HOME}/Library/Python/3.12/bin:${HOME}/Library/Python/3.11/bin:${HOME}/Library/Python/3.10/bin:${PATH}"
+  if command -v snapshot >/dev/null 2>&1; then
+    info "Installed: $(snapshot --version 2>/dev/null || snapshot --help | head -1)"
+    info "Run: snapshot https://example.com ./mirror"
+    return 0
+  fi
+  warn "snapshot installed, but not on PATH."
+  warn "Add this to your shell profile:"
+  echo "  export PATH=\"\${HOME}/.local/bin:\${PATH}\""
+  return 1
+}
+
+main() {
+  info "Installing snapshot from ${INSTALL_SPEC}"
+
+  if ! PY="$(find_python)"; then
+    err "Python 3.10+ is required."
+    err "Install Python from https://www.python.org/downloads/ and re-run this script."
+    exit 1
+  fi
+
+  if ! ensure_pip "$PY"; then
+    err "pip is required but could not be installed."
+    exit 1
+  fi
+
+  if ! install_with_pipx "$PY"; then
+    install_with_pip "$PY"
+  fi
+
+  verify_install || true
+}
+
+main "$@"