web-snapshot-cli 1.2.0__tar.gz → 1.3.0__tar.gz

{web_snapshot_cli-1.2.0 → web_snapshot_cli-1.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: web-snapshot-cli
-Version: 1.2.0
+Version: 1.3.0
 Summary: Take offline snapshots of any website and restore them locally
 Project-URL: Homepage, https://github.com/codingsushi79/Snapshot
 Project-URL: Repository, https://github.com/codingsushi79/Snapshot

{web_snapshot_cli-1.2.0 → web_snapshot_cli-1.3.0}/docs/index.html

@@ -65,7 +65,7 @@
       <div class="field snapshot-only">
         <label for="wordlists">Extra wordlists (one per line)</label>
         <textarea id="wordlists" placeholder="common&#10;large&#10;/path/to/SecLists/Discovery/Web-Content/common.txt"></textarea>
-        <div class="hint">Builtin names: <code>common</code>, <code>large</code>. --gobuster uses both by default. Any page whose full text contains <code>404</code> is skipped, even on HTTP 200.</div>
+        <div class="hint">Builtin names: <code>common</code>, <code>large</code>. Probes follow redirects and skip any page whose text contains <code>404</code> or returns HTTP 404.</div>
       </div>
       <div class="field snapshot-only">
         <label for="wordlistExt">Wordlist extensions</label>

{web_snapshot_cli-1.2.0 → web_snapshot_cli-1.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "web-snapshot-cli"
-version = "1.2.0"
+version = "1.3.0"
 description = "Take offline snapshots of any website and restore them locally"
 readme = "README.md"
 license = "MIT"

{web_snapshot_cli-1.2.0 → web_snapshot_cli-1.3.0}/src/snapshot/__init__.py

@@ -1,3 +1,3 @@
 """snapshot — offline website snapshots."""
 
-__version__ = "1.2.0"
+__version__ = "1.3.0"

{web_snapshot_cli-1.2.0 → web_snapshot_cli-1.3.0}/src/snapshot/downloader.py

@@ -364,6 +364,13 @@ class SnapshotEngine:
 
         try:
             response = await self._fetch(url)
+        except httpx.HTTPStatusError as exc:
+            if exc.response.status_code in {404, 410}:
+                self._pages_skipped += 1
+                self._log(f"skip (404): {url}")
+                return []
+            self._errors.append(f"{url}: {exc}")
+            return []
         except Exception as exc:  # noqa: BLE001
             self._errors.append(f"{url}: {exc}")
             return []

{web_snapshot_cli-1.2.0 → web_snapshot_cli-1.3.0}/src/snapshot/wordlist.py

@@ -12,7 +12,7 @@ from snapshot.crawl_policy import RobotsChecker
 from snapshot.utils import is_not_found_page
 
 BUILTIN_WORDLISTS = ("common", "large")
-HIT_STATUS = {200, 201, 204, 301, 302, 307, 308, 401, 403}
+FOUND_STATUS = {200, 201, 204, 401, 403}
 
 
 def resolve_wordlist_sources(
@@ -157,16 +157,11 @@ def normalize_probe_url(base: str, path: str) -> str | None:
 
 
 async def _probe_url(client: httpx.AsyncClient, url: str) -> bool:
-    for method in ("HEAD", "GET"):
-        try:
-            response = await client.request(method, url, follow_redirects=False)
-        except httpx.RequestError:
-            return False
-        if response.status_code not in HIT_STATUS:
-            if response.status_code == 405 and method == "HEAD":
-                continue
-            return False
-        if method == "HEAD":
-            continue
-        return not is_not_found_page(response)
-    return False
+    """Probe a URL the same way page snapshots are fetched."""
+    try:
+        response = await client.get(url, follow_redirects=True)
+    except httpx.RequestError:
+        return False
+    if response.status_code not in FOUND_STATUS:
+        return False
+    return not is_not_found_page(response)

{web_snapshot_cli-1.2.0 → web_snapshot_cli-1.3.0}/tests/test_crawler.py

@@ -1,6 +1,7 @@
 from pathlib import Path
 from unittest.mock import AsyncMock, MagicMock, patch
 
+import httpx
 import pytest
 from bs4 import BeautifulSoup
 
@@ -93,3 +94,29 @@ async def test_snapshot_skips_soft_404_page():
     assert result.pages_saved == 0
     assert result.pages_skipped == 1
     assert "https://example.com/" not in engine._seen_pages
+    assert result.errors == []
+
+
+@pytest.mark.asyncio
+async def test_snapshot_skips_http_404_without_error():
+    async def fake_fetch(url: str):
+        response = MagicMock()
+        response.raise_for_status.side_effect = httpx.HTTPStatusError(
+            "not found",
+            request=MagicMock(),
+            response=MagicMock(status_code=404),
+        )
+        raise response.raise_for_status.side_effect
+
+    engine = SnapshotEngine(
+        "https://example.com/",
+        Path("/tmp/http-404-test"),
+        SnapshotOptions(crawl=False, download_assets=False),
+    )
+
+    with patch.object(engine, "_fetch", new=AsyncMock(side_effect=fake_fetch)):
+        result = await engine.run()
+
+    assert result.pages_saved == 0
+    assert result.pages_skipped == 1
+    assert result.errors == []

web_snapshot_cli-1.3.0/tests/test_wordlist.py

@@ -0,0 +1,92 @@
+from unittest.mock import AsyncMock, MagicMock
+
+import httpx
+import pytest
+
+from snapshot.wordlist import _probe_url, paths_for_word, resolve_wordlist_sources
+
+
+def test_paths_for_word_file_entry():
+    paths = paths_for_word("robots.txt", ())
+    assert paths == ["/robots.txt"]
+
+
+def test_paths_for_word_directory_entry():
+    paths = paths_for_word("admin", (".html",))
+    assert "/admin" in paths
+    assert "/admin/" in paths
+    assert "/admin.html" in paths
+
+
+def test_resolve_wordlist_sources_gobuster_defaults():
+    assert resolve_wordlist_sources(True, []) == ["common", "large"]
+
+
+def test_resolve_wordlist_sources_custom():
+    assert resolve_wordlist_sources(False, ["common", "/tmp/list.txt"]) == [
+        "common",
+        "/tmp/list.txt",
+    ]
+
+
+def test_load_builtin_common():
+    from snapshot.wordlist import load_words
+
+    words = load_words(["common"])
+    assert "robots.txt" in words
+    assert "sitemap.xml" in words
+
+
+@pytest.mark.asyncio
+async def test_probe_url_rejects_soft_404():
+    client = MagicMock(spec=httpx.AsyncClient)
+    response = MagicMock()
+    response.status_code = 200
+    response.headers = {"content-type": "text/html"}
+    response.url = "https://example.com/missing"
+    response.text = (
+        "<html><head><title>404 Not Found</title></head><body><h1>404</h1></body></html>"
+    )
+    client.get = AsyncMock(return_value=response)
+
+    assert await _probe_url(client, "https://example.com/missing") is False
+    client.get.assert_awaited_once_with("https://example.com/missing", follow_redirects=True)
+
+
+@pytest.mark.asyncio
+async def test_probe_url_accepts_valid_page():
+    client = MagicMock(spec=httpx.AsyncClient)
+    response = MagicMock()
+    response.status_code = 200
+    response.headers = {"content-type": "text/html"}
+    response.url = "https://example.com/about"
+    response.text = "<html><head><title>About</title></head><body><h1>About</h1></body></html>"
+    client.get = AsyncMock(return_value=response)
+
+    assert await _probe_url(client, "https://example.com/about") is True
+
+
+@pytest.mark.asyncio
+async def test_probe_url_rejects_hard_404_after_redirect():
+    client = MagicMock(spec=httpx.AsyncClient)
+    response = MagicMock()
+    response.status_code = 404
+    response.headers = {"content-type": "text/html"}
+    response.url = "https://example.com/missing"
+    response.text = "<html><body>missing</body></html>"
+    client.get = AsyncMock(return_value=response)
+
+    assert await _probe_url(client, "https://example.com/missing/") is False
+
+
+@pytest.mark.asyncio
+async def test_probe_url_does_not_count_redirect_without_final_page():
+    client = MagicMock(spec=httpx.AsyncClient)
+    response = MagicMock()
+    response.status_code = 308
+    response.headers = {"content-type": "text/html"}
+    response.url = "https://example.com/old/"
+    response.text = ""
+    client.get = AsyncMock(return_value=response)
+
+    assert await _probe_url(client, "https://example.com/old/") is False

web_snapshot_cli-1.2.0/tests/test_wordlist.py

@@ -1,69 +0,0 @@
-from unittest.mock import AsyncMock, MagicMock
-
-import httpx
-import pytest
-
-from snapshot.wordlist import _probe_url, paths_for_word, resolve_wordlist_sources
-
-
-def test_paths_for_word_file_entry():
-    paths = paths_for_word("robots.txt", ())
-    assert paths == ["/robots.txt"]
-
-
-def test_paths_for_word_directory_entry():
-    paths = paths_for_word("admin", (".html",))
-    assert "/admin" in paths
-    assert "/admin/" in paths
-    assert "/admin.html" in paths
-
-
-def test_resolve_wordlist_sources_gobuster_defaults():
-    assert resolve_wordlist_sources(True, []) == ["common", "large"]
-
-
-def test_resolve_wordlist_sources_custom():
-    assert resolve_wordlist_sources(False, ["common", "/tmp/list.txt"]) == [
-        "common",
-        "/tmp/list.txt",
-    ]
-
-
-def test_load_builtin_common():
-    from snapshot.wordlist import load_words
-
-    words = load_words(["common"])
-    assert "robots.txt" in words
-    assert "sitemap.xml" in words
-
-
-@pytest.mark.asyncio
-async def test_probe_url_rejects_soft_404():
-    client = MagicMock(spec=httpx.AsyncClient)
-    head_response = MagicMock()
-    head_response.status_code = 200
-    get_response = MagicMock()
-    get_response.status_code = 200
-    get_response.headers = {"content-type": "text/html"}
-    get_response.url = "https://example.com/missing"
-    get_response.text = (
-        "<html><head><title>404 Not Found</title></head><body><h1>404</h1></body></html>"
-    )
-    client.request = AsyncMock(side_effect=[head_response, get_response])
-
-    assert await _probe_url(client, "https://example.com/missing") is False
-
-
-@pytest.mark.asyncio
-async def test_probe_url_accepts_valid_page():
-    client = MagicMock(spec=httpx.AsyncClient)
-    head_response = MagicMock()
-    head_response.status_code = 200
-    get_response = MagicMock()
-    get_response.status_code = 200
-    get_response.headers = {"content-type": "text/html"}
-    get_response.url = "https://example.com/about"
-    get_response.text = "<html><head><title>About</title></head><body><h1>About</h1></body></html>"
-    client.request = AsyncMock(side_effect=[head_response, get_response])
-
-    assert await _probe_url(client, "https://example.com/about") is True