web-snapshot-cli 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {web_snapshot_cli-0.1.0 → web_snapshot_cli-0.2.0}/.github/workflows/publish.yml +3 -2
- {web_snapshot_cli-0.1.0 → web_snapshot_cli-0.2.0}/PKG-INFO +2 -37
- {web_snapshot_cli-0.1.0 → web_snapshot_cli-0.2.0}/README.md +1 -36
- {web_snapshot_cli-0.1.0 → web_snapshot_cli-0.2.0}/pyproject.toml +1 -1
- {web_snapshot_cli-0.1.0 → web_snapshot_cli-0.2.0}/src/snapshot/__init__.py +1 -1
- {web_snapshot_cli-0.1.0 → web_snapshot_cli-0.2.0}/src/snapshot/cli.py +6 -3
- {web_snapshot_cli-0.1.0 → web_snapshot_cli-0.2.0}/src/snapshot/downloader.py +1 -1
- web_snapshot_cli-0.1.0/install.ps1 +0 -67
- web_snapshot_cli-0.1.0/install.sh +0 -109
- {web_snapshot_cli-0.1.0 → web_snapshot_cli-0.2.0}/.github/workflows/ci.yml +0 -0
- {web_snapshot_cli-0.1.0 → web_snapshot_cli-0.2.0}/.gitignore +0 -0
- {web_snapshot_cli-0.1.0 → web_snapshot_cli-0.2.0}/LICENSE +0 -0
- {web_snapshot_cli-0.1.0 → web_snapshot_cli-0.2.0}/src/snapshot/__main__.py +0 -0
- {web_snapshot_cli-0.1.0 → web_snapshot_cli-0.2.0}/src/snapshot/crawl_policy.py +0 -0
- {web_snapshot_cli-0.1.0 → web_snapshot_cli-0.2.0}/src/snapshot/manifest.py +0 -0
- {web_snapshot_cli-0.1.0 → web_snapshot_cli-0.2.0}/src/snapshot/restore.py +0 -0
- {web_snapshot_cli-0.1.0 → web_snapshot_cli-0.2.0}/src/snapshot/utils.py +0 -0
- {web_snapshot_cli-0.1.0 → web_snapshot_cli-0.2.0}/tests/test_crawl_policy.py +0 -0
- {web_snapshot_cli-0.1.0 → web_snapshot_cli-0.2.0}/tests/test_crawler.py +0 -0
- {web_snapshot_cli-0.1.0 → web_snapshot_cli-0.2.0}/tests/test_features.py +0 -0
- {web_snapshot_cli-0.1.0 → web_snapshot_cli-0.2.0}/tests/test_utils.py +0 -0
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
# PyPI
|
|
2
|
-
# Trusted publisher: owner codingsushi79, repo Snapshot, workflow publish.yml, env pypi
|
|
1
|
+
# PyPI: web-snapshot-cli
|
|
3
2
|
name: Publish to PyPI
|
|
4
3
|
|
|
5
4
|
on:
|
|
@@ -62,3 +61,5 @@ jobs:
|
|
|
62
61
|
path: dist/
|
|
63
62
|
|
|
64
63
|
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
64
|
+
with:
|
|
65
|
+
skip-existing: true
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: web-snapshot-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Take offline snapshots of any website and restore them locally
|
|
5
5
|
Project-URL: Homepage, https://github.com/codingsushi79/Snapshot
|
|
6
6
|
Project-URL: Repository, https://github.com/codingsushi79/Snapshot
|
|
@@ -33,25 +33,7 @@ Take a fast offline copy of any public website, then serve it locally.
|
|
|
33
33
|
|
|
34
34
|
## Install
|
|
35
35
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
```bash
|
|
39
|
-
curl -fsSL https://raw.githubusercontent.com/codingsushi79/Snapshot/main/install.sh | bash
|
|
40
|
-
```
|
|
41
|
-
|
|
42
|
-
**One-liner (Windows PowerShell):**
|
|
43
|
-
|
|
44
|
-
```powershell
|
|
45
|
-
irm https://raw.githubusercontent.com/codingsushi79/Snapshot/main/install.ps1 | iex
|
|
46
|
-
```
|
|
47
|
-
|
|
48
|
-
**Universal fallback** (requires Python 3.10+):
|
|
49
|
-
|
|
50
|
-
```bash
|
|
51
|
-
python3 -m pip install --user git+https://github.com/codingsushi79/Snapshot.git
|
|
52
|
-
```
|
|
53
|
-
|
|
54
|
-
**From PyPI** (once published):
|
|
36
|
+
Requires Python 3.10+.
|
|
55
37
|
|
|
56
38
|
```bash
|
|
57
39
|
pip install web-snapshot-cli
|
|
@@ -63,23 +45,6 @@ Then run:
|
|
|
63
45
|
snapshot https://example.com ./mirror
|
|
64
46
|
```
|
|
65
47
|
|
|
66
|
-
<details>
|
|
67
|
-
<summary>Other install methods</summary>
|
|
68
|
-
|
|
69
|
-
```bash
|
|
70
|
-
# With pipx (recommended for CLI tools)
|
|
71
|
-
pipx install web-snapshot-cli
|
|
72
|
-
# or from git:
|
|
73
|
-
pipx install git+https://github.com/codingsushi79/Snapshot.git
|
|
74
|
-
|
|
75
|
-
# From a local clone
|
|
76
|
-
git clone https://github.com/codingsushi79/Snapshot.git
|
|
77
|
-
cd Snapshot
|
|
78
|
-
pip install -e .
|
|
79
|
-
```
|
|
80
|
-
|
|
81
|
-
</details>
|
|
82
|
-
|
|
83
48
|
## Usage
|
|
84
49
|
|
|
85
50
|
### Snapshot a single page
|
|
@@ -4,25 +4,7 @@ Take a fast offline copy of any public website, then serve it locally.
|
|
|
4
4
|
|
|
5
5
|
## Install
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
```bash
|
|
10
|
-
curl -fsSL https://raw.githubusercontent.com/codingsushi79/Snapshot/main/install.sh | bash
|
|
11
|
-
```
|
|
12
|
-
|
|
13
|
-
**One-liner (Windows PowerShell):**
|
|
14
|
-
|
|
15
|
-
```powershell
|
|
16
|
-
irm https://raw.githubusercontent.com/codingsushi79/Snapshot/main/install.ps1 | iex
|
|
17
|
-
```
|
|
18
|
-
|
|
19
|
-
**Universal fallback** (requires Python 3.10+):
|
|
20
|
-
|
|
21
|
-
```bash
|
|
22
|
-
python3 -m pip install --user git+https://github.com/codingsushi79/Snapshot.git
|
|
23
|
-
```
|
|
24
|
-
|
|
25
|
-
**From PyPI** (once published):
|
|
7
|
+
Requires Python 3.10+.
|
|
26
8
|
|
|
27
9
|
```bash
|
|
28
10
|
pip install web-snapshot-cli
|
|
@@ -34,23 +16,6 @@ Then run:
|
|
|
34
16
|
snapshot https://example.com ./mirror
|
|
35
17
|
```
|
|
36
18
|
|
|
37
|
-
<details>
|
|
38
|
-
<summary>Other install methods</summary>
|
|
39
|
-
|
|
40
|
-
```bash
|
|
41
|
-
# With pipx (recommended for CLI tools)
|
|
42
|
-
pipx install web-snapshot-cli
|
|
43
|
-
# or from git:
|
|
44
|
-
pipx install git+https://github.com/codingsushi79/Snapshot.git
|
|
45
|
-
|
|
46
|
-
# From a local clone
|
|
47
|
-
git clone https://github.com/codingsushi79/Snapshot.git
|
|
48
|
-
cd Snapshot
|
|
49
|
-
pip install -e .
|
|
50
|
-
```
|
|
51
|
-
|
|
52
|
-
</details>
|
|
53
|
-
|
|
54
19
|
## Usage
|
|
55
20
|
|
|
56
21
|
### Snapshot a single page
|
|
@@ -180,6 +180,7 @@ def _build_options(
|
|
|
180
180
|
)
|
|
181
181
|
@click.option(
|
|
182
182
|
"--sitemap",
|
|
183
|
+
"use_sitemap",
|
|
183
184
|
is_flag=True,
|
|
184
185
|
help="Seed crawl URLs from sitemap.xml and robots.txt Sitemap directives.",
|
|
185
186
|
)
|
|
@@ -198,7 +199,9 @@ def _build_options(
|
|
|
198
199
|
@click.option("--host", default="127.0.0.1", show_default=True, help="Host for -restore.")
|
|
199
200
|
@click.option("--no-open", is_flag=True, help="Do not open a browser when restoring.")
|
|
200
201
|
@click.version_option(__version__, prog_name="snapshot")
|
|
202
|
+
@click.pass_context
|
|
201
203
|
def main(
|
|
204
|
+
ctx: click.Context,
|
|
202
205
|
url: str | None,
|
|
203
206
|
output_dir: str | None,
|
|
204
207
|
extra_args: tuple[str, ...],
|
|
@@ -245,9 +248,9 @@ def main(
|
|
|
245
248
|
return
|
|
246
249
|
|
|
247
250
|
if not url or not output_dir:
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
+
if not url:
|
|
252
|
+
ctx.fail("Missing argument 'URL'.")
|
|
253
|
+
ctx.fail("Missing argument 'OUTPUT_DIR'.")
|
|
251
254
|
|
|
252
255
|
options = _build_options(
|
|
253
256
|
extras,
|
|
@@ -39,7 +39,7 @@ class SnapshotOptions:
|
|
|
39
39
|
timeout: float = 15.0
|
|
40
40
|
concurrency: int = 16
|
|
41
41
|
same_origin_only: bool = True
|
|
42
|
-
user_agent: str = "web-snapshot-cli/0.
|
|
42
|
+
user_agent: str = "web-snapshot-cli/0.2 (+https://github.com/codingsushi79/Snapshot)"
|
|
43
43
|
cookies: list[str] = field(default_factory=list)
|
|
44
44
|
headers: list[str] = field(default_factory=list)
|
|
45
45
|
include_patterns: list[str] = field(default_factory=list)
|
|
@@ -1,67 +0,0 @@
|
|
|
1
|
-
#Requires -Version 5.1
|
|
2
|
-
$ErrorActionPreference = "Stop"
|
|
3
|
-
|
|
4
|
-
$Repo = if ($env:SNAPSHOT_REPO) { $env:SNAPSHOT_REPO } else { "https://github.com/codingsushi79/Snapshot.git" }
|
|
5
|
-
$Ref = if ($env:SNAPSHOT_REF) { $env:SNAPSHOT_REF } else { "main" }
|
|
6
|
-
$InstallSpec = "git+$Repo@$Ref"
|
|
7
|
-
|
|
8
|
-
function Write-Info($msg) { Write-Host "→ $msg" -ForegroundColor Cyan }
|
|
9
|
-
function Write-Warn($msg) { Write-Host "! $msg" -ForegroundColor Yellow }
|
|
10
|
-
function Write-Err($msg) { Write-Host "✗ $msg" -ForegroundColor Red }
|
|
11
|
-
|
|
12
|
-
function Find-Python {
|
|
13
|
-
foreach ($cmd in @("python", "python3", "py")) {
|
|
14
|
-
if (Get-Command $cmd -ErrorAction SilentlyContinue) {
|
|
15
|
-
$ok = & $cmd -c "import sys; raise SystemExit(0 if sys.version_info >= (3,10) else 1)" 2>$null
|
|
16
|
-
if ($LASTEXITCODE -eq 0) { return $cmd }
|
|
17
|
-
}
|
|
18
|
-
}
|
|
19
|
-
return $null
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
function Ensure-Pip($py) {
|
|
23
|
-
& $py -m pip --version 2>$null | Out-Null
|
|
24
|
-
if ($LASTEXITCODE -ne 0) {
|
|
25
|
-
Write-Info "Bootstrapping pip…"
|
|
26
|
-
& $py -m ensurepip --upgrade 2>$null | Out-Null
|
|
27
|
-
}
|
|
28
|
-
& $py -m pip --version 2>$null | Out-Null
|
|
29
|
-
return ($LASTEXITCODE -eq 0)
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
Write-Info "Installing snapshot from $InstallSpec"
|
|
33
|
-
|
|
34
|
-
$py = Find-Python
|
|
35
|
-
if (-not $py) {
|
|
36
|
-
Write-Err "Python 3.10+ is required."
|
|
37
|
-
Write-Err "Install from https://www.python.org/downloads/ and re-run this script."
|
|
38
|
-
exit 1
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
if (-not (Ensure-Pip $py)) {
|
|
42
|
-
Write-Err "pip is required but could not be installed."
|
|
43
|
-
exit 1
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
if (Get-Command pipx -ErrorAction SilentlyContinue) {
|
|
47
|
-
Write-Info "Installing snapshot with pipx…"
|
|
48
|
-
pipx install --force $InstallSpec
|
|
49
|
-
} else {
|
|
50
|
-
Write-Info "Installing snapshot with pip…"
|
|
51
|
-
& $py -m pip install --user --upgrade $InstallSpec
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
$localBin = Join-Path $env:USERPROFILE ".local\\bin"
|
|
55
|
-
$scriptsDir = Join-Path $env:APPDATA "Python\\Python312\\Scripts"
|
|
56
|
-
$paths = @($localBin, $scriptsDir) | Where-Object { Test-Path $_ }
|
|
57
|
-
if ($paths.Count -gt 0) {
|
|
58
|
-
$env:PATH = ($paths -join ";") + ";" + $env:PATH
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
if (Get-Command snapshot -ErrorAction SilentlyContinue) {
|
|
62
|
-
Write-Info "Installed snapshot."
|
|
63
|
-
Write-Info "Run: snapshot https://example.com ./mirror"
|
|
64
|
-
} else {
|
|
65
|
-
Write-Warn "snapshot installed, but not on PATH."
|
|
66
|
-
Write-Warn "Restart your terminal or add Python Scripts to PATH."
|
|
67
|
-
}
|
|
@@ -1,109 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
set -euo pipefail
|
|
3
|
-
|
|
4
|
-
REPO="${SNAPSHOT_REPO:-https://github.com/codingsushi79/Snapshot.git}"
|
|
5
|
-
REF="${SNAPSHOT_REF:-main}"
|
|
6
|
-
INSTALL_SPEC="git+${REPO}@${REF}"
|
|
7
|
-
|
|
8
|
-
info() { printf '\033[1;34m→\033[0m %s\n' "$*"; }
|
|
9
|
-
warn() { printf '\033[1;33m!\033[0m %s\n' "$*"; }
|
|
10
|
-
err() { printf '\033[1;31m✗\033[0m %s\n' "$*" >&2; }
|
|
11
|
-
|
|
12
|
-
need_cmd() {
|
|
13
|
-
command -v "$1" >/dev/null 2>&1
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
find_python() {
|
|
17
|
-
for cmd in python3 python py; do
|
|
18
|
-
if need_cmd "$cmd"; then
|
|
19
|
-
if "$cmd" -c 'import sys; raise SystemExit(0 if sys.version_info >= (3, 10) else 1)' 2>/dev/null; then
|
|
20
|
-
echo "$cmd"
|
|
21
|
-
return 0
|
|
22
|
-
fi
|
|
23
|
-
fi
|
|
24
|
-
done
|
|
25
|
-
return 1
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
ensure_pip() {
|
|
29
|
-
local py="$1"
|
|
30
|
-
if "$py" -m pip --version >/dev/null 2>&1; then
|
|
31
|
-
return 0
|
|
32
|
-
fi
|
|
33
|
-
info "Bootstrapping pip…"
|
|
34
|
-
"$py" -m ensurepip --upgrade >/dev/null 2>&1 || true
|
|
35
|
-
"$py" -m pip --version >/dev/null 2>&1
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
pip_user_flag() {
|
|
39
|
-
if [[ -n "${VIRTUAL_ENV:-}" ]]; then
|
|
40
|
-
echo ""
|
|
41
|
-
else
|
|
42
|
-
echo "--user"
|
|
43
|
-
fi
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
install_with_pipx() {
|
|
47
|
-
local py="$1"
|
|
48
|
-
info "Installing snapshot with pipx…"
|
|
49
|
-
if ! need_cmd pipx; then
|
|
50
|
-
local user_flag
|
|
51
|
-
user_flag="$(pip_user_flag)"
|
|
52
|
-
# shellcheck disable=SC2086
|
|
53
|
-
"$py" -m pip install $user_flag pipx
|
|
54
|
-
export PATH="${HOME}/.local/bin:${PATH}"
|
|
55
|
-
if need_cmd pipx; then
|
|
56
|
-
pipx ensurepath >/dev/null 2>&1 || true
|
|
57
|
-
fi
|
|
58
|
-
fi
|
|
59
|
-
if need_cmd pipx; then
|
|
60
|
-
pipx install --force "$INSTALL_SPEC"
|
|
61
|
-
return 0
|
|
62
|
-
fi
|
|
63
|
-
return 1
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
install_with_pip() {
|
|
67
|
-
local py="$1"
|
|
68
|
-
local user_flag
|
|
69
|
-
user_flag="$(pip_user_flag)"
|
|
70
|
-
info "Installing snapshot with pip…"
|
|
71
|
-
# shellcheck disable=SC2086
|
|
72
|
-
"$py" -m pip install $user_flag --upgrade "$INSTALL_SPEC"
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
verify_install() {
|
|
76
|
-
export PATH="${HOME}/.local/bin:${HOME}/Library/Python/3.12/bin:${HOME}/Library/Python/3.11/bin:${HOME}/Library/Python/3.10/bin:${PATH}"
|
|
77
|
-
if command -v snapshot >/dev/null 2>&1; then
|
|
78
|
-
info "Installed: $(snapshot --version 2>/dev/null || snapshot --help | head -1)"
|
|
79
|
-
info "Run: snapshot https://example.com ./mirror"
|
|
80
|
-
return 0
|
|
81
|
-
fi
|
|
82
|
-
warn "snapshot installed, but not on PATH."
|
|
83
|
-
warn "Add this to your shell profile:"
|
|
84
|
-
echo " export PATH=\"\${HOME}/.local/bin:\${PATH}\""
|
|
85
|
-
return 1
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
main() {
|
|
89
|
-
info "Installing snapshot from ${INSTALL_SPEC}"
|
|
90
|
-
|
|
91
|
-
if ! PY="$(find_python)"; then
|
|
92
|
-
err "Python 3.10+ is required."
|
|
93
|
-
err "Install Python from https://www.python.org/downloads/ and re-run this script."
|
|
94
|
-
exit 1
|
|
95
|
-
fi
|
|
96
|
-
|
|
97
|
-
if ! ensure_pip "$PY"; then
|
|
98
|
-
err "pip is required but could not be installed."
|
|
99
|
-
exit 1
|
|
100
|
-
fi
|
|
101
|
-
|
|
102
|
-
if ! install_with_pipx "$PY"; then
|
|
103
|
-
install_with_pip "$PY"
|
|
104
|
-
fi
|
|
105
|
-
|
|
106
|
-
verify_install || true
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
main "$@"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|