web-snapshot-cli 1.1.0__tar.gz → 1.3.0__tar.gz

{web_snapshot_cli-1.1.0 → web_snapshot_cli-1.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: web-snapshot-cli
-Version: 1.1.0
+Version: 1.3.0
 Summary: Take offline snapshots of any website and restore them locally
 Project-URL: Homepage, https://github.com/codingsushi79/Snapshot
 Project-URL: Repository, https://github.com/codingsushi79/Snapshot

{web_snapshot_cli-1.1.0 → web_snapshot_cli-1.3.0}/docs/index.html

@@ -50,7 +50,7 @@
         </label>
         <label class="check-item">
           <input type="checkbox" id="gobuster" />
-          <div>--gobuster<span>Bundled wordlists; soft 404 pages are filtered out</span></div>
+          <div>--gobuster<span>Wordlist discovery; skips any page mentioning 404</span></div>
         </label>
         <label class="check-item">
           <input type="checkbox" id="sitemap" />
@@ -65,7 +65,7 @@
       <div class="field snapshot-only">
         <label for="wordlists">Extra wordlists (one per line)</label>
         <textarea id="wordlists" placeholder="common&#10;large&#10;/path/to/SecLists/Discovery/Web-Content/common.txt"></textarea>
-        <div class="hint">Builtin names: <code>common</code>, <code>large</code>. --gobuster uses both by default. Pages that return 200 but show 404/not-found content are skipped.</div>
+        <div class="hint">Builtin names: <code>common</code>, <code>large</code>. Probes follow redirects and skip any page whose text contains <code>404</code> or returns HTTP 404.</div>
       </div>
       <div class="field snapshot-only">
         <label for="wordlistExt">Wordlist extensions</label>
@@ -143,7 +143,7 @@
         </label>
         <label class="check-item">
           <input type="checkbox" id="noAssets" />
-          <div>--no-assets<span>Skip CSS/JS/images</span></div>
+          <div>--no-assets<span>Skip CSS/JS/images/animations</span></div>
         </label>
         <label class="check-item">
           <input type="checkbox" id="verbose" />

{web_snapshot_cli-1.1.0 → web_snapshot_cli-1.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "web-snapshot-cli"
-version = "1.1.0"
+version = "1.3.0"
 description = "Take offline snapshots of any website and restore them locally"
 readme = "README.md"
 license = "MIT"

{web_snapshot_cli-1.1.0 → web_snapshot_cli-1.3.0}/src/snapshot/__init__.py

@@ -1,3 +1,3 @@
 """snapshot — offline website snapshots."""
 
-__version__ = "1.1.0"
+__version__ = "1.3.0"

{web_snapshot_cli-1.1.0 → web_snapshot_cli-1.3.0}/src/snapshot/downloader.py

@@ -20,6 +20,7 @@ from snapshot.manifest import MANIFEST_NAME, SnapshotManifest
 from snapshot.utils import (
     SRCSET_ATTRS,
     URL_ATTRS,
+    extract_css_urls,
     is_not_found_page,
     normalize_url,
     page_local_path,
@@ -363,6 +364,13 @@ class SnapshotEngine:
 
         try:
             response = await self._fetch(url)
+        except httpx.HTTPStatusError as exc:
+            if exc.response.status_code in {404, 410}:
+                self._pages_skipped += 1
+                self._log(f"skip (404): {url}")
+                return []
+            self._errors.append(f"{url}: {exc}")
+            return []
         except Exception as exc:  # noqa: BLE001
             self._errors.append(f"{url}: {exc}")
             return []
@@ -419,12 +427,15 @@ class SnapshotEngine:
                         absolute = normalize_url(piece, page_url)
                         if absolute and not self._is_page_url(absolute):
                             urls.add(absolute)
-        for style in soup.find_all("style"):
-            if style.string:
-                for match in re.findall(r"url\(([^)]+)\)", style.string):
+            style = tag.get("style")
+            if style:
+                for match in re.findall(r"url\(([^)]+)\)", style):
                     absolute = normalize_url(match.strip("'\""), page_url)
                     if absolute and not self._is_page_url(absolute):
                         urls.add(absolute)
+        for style in soup.find_all("style"):
+            if style.string:
+                urls.update(extract_css_urls(style.string, page_url))
         return urls
 
     def _collect_page_links(self, soup: BeautifulSoup, page_url: str) -> list[str]:
@@ -466,7 +477,7 @@ class SnapshotEngine:
                     lambda asset_url: self._css_mapper(asset_url, local_path),
                 )
                 local_path.write_text(css, encoding="utf-8")
-                nested = self._extract_css_urls(css, url)
+                nested = extract_css_urls(css, url)
                 await asyncio.gather(*(self._download_asset(u) for u in nested))
             else:
                 local_path.write_bytes(response.content)
@@ -480,14 +491,6 @@ class SnapshotEngine:
             self._url_to_path[asset_url] = target
         return relative_href(css_path, target)
 
-    def _extract_css_urls(self, css: str, base_url: str) -> set[str]:
-        urls: set[str] = set()
-        for match in re.findall(r"url\(([^)]+)\)", css):
-            absolute = normalize_url(match.strip("'\""), base_url)
-            if absolute:
-                urls.add(absolute)
-        return urls
-
     def _rewrite_html(self, soup: BeautifulSoup, page_url: str, page_path: Path) -> None:
         for tag in soup.find_all(True):
             for attr in URL_ATTRS:
@@ -533,6 +536,8 @@ class SnapshotEngine:
     def _render_page(self, soup: BeautifulSoup) -> str:
         if self.options.lang == "md":
             return self._html_to_markdown(soup)
+        if not soup.find("html"):
+            return f"<!DOCTYPE html>\n{str(soup)}"
         return str(soup)
 
     def _html_to_markdown(self, soup: BeautifulSoup) -> str:

{web_snapshot_cli-1.1.0 → web_snapshot_cli-1.3.0}/src/snapshot/utils.py

@@ -12,8 +12,28 @@ if TYPE_CHECKING:
     import httpx
 
 # Attributes that may contain fetchable URLs.
-URL_ATTRS = ("href", "src", "poster", "data-src", "data-background")
+URL_ATTRS = (
+    "href",
+    "src",
+    "poster",
+    "data-src",
+    "data-background",
+    "data-lazy-src",
+    "data-original",
+    "data-lazy",
+    "data-bg",
+    "data-background-image",
+    "data-image",
+    "data-href",
+    "data-url",
+    "data-video",
+    "data-poster",
+    "data-anim-src",
+    "data-animation",
+)
 SRCSET_ATTRS = ("srcset", "data-srcset")
+TEXT_ATTRS = ("aria-label", "alt", "title", "placeholder")
+_SKIP_TAGS = frozenset({"script", "style", "noscript"})
 
 # Tags treated as HTML pages when crawled.
 PAGE_EXTENSIONS = {".html", ".htm", ".xhtml", ""}
@@ -43,14 +63,44 @@ ASSET_EXTENSIONS = {
 
 SKIP_SCHEMES = {"mailto", "tel", "javascript", "data", "blob", "about", "file"}
 
-_NOT_FOUND_TEXT = re.compile(
-    r"\b404\b|not\s+found|page\s+not\s+found|page\s+cannot\s+be\s+found",
-    re.IGNORECASE,
-)
+_404_MARKER = re.compile(r"404", re.IGNORECASE)
+
+
+def extract_page_text(html: str) -> str:
+    """Collect visible and semantic text from an HTML document."""
+    from bs4 import BeautifulSoup
+
+    soup = BeautifulSoup(html, "html.parser")
+    parts: list[str] = []
+
+    if soup.title:
+        parts.append(soup.title.get_text(" ", strip=True))
+
+    for meta in soup.find_all("meta"):
+        content = meta.get("content")
+        if content:
+            parts.append(str(content))
+
+    for tag in soup.find_all(True):
+        for attr in TEXT_ATTRS:
+            value = tag.get(attr)
+            if value:
+                parts.append(str(value))
+
+    for skip_tag in list(soup.find_all(_SKIP_TAGS)):
+        skip_tag.decompose()
+
+    parts.append(soup.get_text(" ", strip=True))
+    return " ".join(part for part in parts if part)
+
+
+def page_mentions_404(text: str) -> bool:
+    """Return True when page text contains a 404 marker."""
+    return bool(_404_MARKER.search(text))
 
 
 def is_not_found_page(response: httpx.Response) -> bool:
-    """Detect hard and soft 404 pages (HTTP 404 or HTML error content with 200 status)."""
+    """Detect hard and soft 404 pages (HTTP 404 or any page text mentioning 404)."""
     if response.status_code == 404:
         return True
     if response.status_code not in {200, 201, 204}:
@@ -62,32 +112,16 @@ def is_not_found_page(response: httpx.Response) -> bool:
         return False
 
     try:
-        text = response.text
+        html = response.text
     except Exception:  # noqa: BLE001
         return False
 
-    from bs4 import BeautifulSoup
-
-    soup = BeautifulSoup(text, "html.parser")
-
-    title = soup.title.get_text(" ", strip=True) if soup.title else ""
-    if title and _NOT_FOUND_TEXT.search(title):
+    if page_mentions_404(extract_page_text(html)):
         return True
 
-    for tag_name in ("h1", "h2"):
-        heading = soup.find(tag_name)
-        if heading:
-            heading_text = heading.get_text(" ", strip=True)
-            if heading_text and _NOT_FOUND_TEXT.search(heading_text):
-                return True
-
-    body = soup.body
-    if body:
-        body_text = body.get_text(" ", strip=True)
-        if body_text and len(body_text) < 500 and _NOT_FOUND_TEXT.search(body_text):
-            return True
-
-    return False
+    stripped = re.sub(r"(?is)<script[^>]*>.*?</script>", " ", html)
+    stripped = re.sub(r"(?is)<style[^>]*>.*?</style>", " ", stripped)
+    return page_mentions_404(stripped)
 
 
 def normalize_url(url: str, base: str | None = None) -> str | None:
@@ -187,6 +221,20 @@ def parse_srcset(value: str) -> list[str]:
     return urls
 
 
+def extract_css_urls(css: str, base_url: str) -> set[str]:
+    """Extract linked asset URLs from CSS, including @import rules."""
+    urls: set[str] = set()
+    for match in re.findall(r"url\(([^)]+)\)", css):
+        absolute = normalize_url(match.strip("'\""), base_url)
+        if absolute:
+            urls.add(absolute)
+    for match in re.findall(r"@import\s+(?:url\()?['\"]?([^'\")\s;]+)", css):
+        absolute = normalize_url(match.strip("'\""), base_url)
+        if absolute:
+            urls.add(absolute)
+    return urls
+
+
 def rewrite_css_urls(css: str, base_url: str, mapper) -> str:
     """Rewrite url(...) references in CSS using mapper(url) -> local path or None."""
 

{web_snapshot_cli-1.1.0 → web_snapshot_cli-1.3.0}/src/snapshot/wordlist.py

@@ -12,7 +12,7 @@ from snapshot.crawl_policy import RobotsChecker
 from snapshot.utils import is_not_found_page
 
 BUILTIN_WORDLISTS = ("common", "large")
-HIT_STATUS = {200, 201, 204, 301, 302, 307, 308, 401, 403}
+FOUND_STATUS = {200, 201, 204, 401, 403}
 
 
 def resolve_wordlist_sources(
@@ -157,16 +157,11 @@ def normalize_probe_url(base: str, path: str) -> str | None:
 
 
 async def _probe_url(client: httpx.AsyncClient, url: str) -> bool:
-    for method in ("HEAD", "GET"):
-        try:
-            response = await client.request(method, url, follow_redirects=False)
-        except httpx.RequestError:
-            return False
-        if response.status_code not in HIT_STATUS:
-            if response.status_code == 405 and method == "HEAD":
-                continue
-            return False
-        if method == "HEAD":
-            continue
-        return not is_not_found_page(response)
-    return False
+    """Probe a URL the same way page snapshots are fetched."""
+    try:
+        response = await client.get(url, follow_redirects=True)
+    except httpx.RequestError:
+        return False
+    if response.status_code not in FOUND_STATUS:
+        return False
+    return not is_not_found_page(response)

{web_snapshot_cli-1.1.0 → web_snapshot_cli-1.3.0}/tests/test_crawler.py

@@ -1,6 +1,7 @@
 from pathlib import Path
 from unittest.mock import AsyncMock, MagicMock, patch
 
+import httpx
 import pytest
 from bs4 import BeautifulSoup
 
@@ -93,3 +94,29 @@ async def test_snapshot_skips_soft_404_page():
     assert result.pages_saved == 0
     assert result.pages_skipped == 1
     assert "https://example.com/" not in engine._seen_pages
+    assert result.errors == []
+
+
+@pytest.mark.asyncio
+async def test_snapshot_skips_http_404_without_error():
+    async def fake_fetch(url: str):
+        response = MagicMock()
+        response.raise_for_status.side_effect = httpx.HTTPStatusError(
+            "not found",
+            request=MagicMock(),
+            response=MagicMock(status_code=404),
+        )
+        raise response.raise_for_status.side_effect
+
+    engine = SnapshotEngine(
+        "https://example.com/",
+        Path("/tmp/http-404-test"),
+        SnapshotOptions(crawl=False, download_assets=False),
+    )
+
+    with patch.object(engine, "_fetch", new=AsyncMock(side_effect=fake_fetch)):
+        result = await engine.run()
+
+    assert result.pages_saved == 0
+    assert result.pages_skipped == 1
+    assert result.errors == []

{web_snapshot_cli-1.1.0 → web_snapshot_cli-1.3.0}/tests/test_utils.py

@@ -1,7 +1,15 @@
 from pathlib import Path
 from unittest.mock import MagicMock
 
-from snapshot.utils import is_not_found_page, normalize_url, page_local_path, url_to_local_path
+from snapshot.utils import (
+    extract_css_urls,
+    extract_page_text,
+    is_not_found_page,
+    normalize_url,
+    page_local_path,
+    page_mentions_404,
+    url_to_local_path,
+)
 
 
 def _html_response(html: str, status_code: int = 200) -> MagicMock:
@@ -23,8 +31,14 @@ def test_is_not_found_page_soft_404_title():
     assert is_not_found_page(_html_response(html)) is True
 
 
-def test_is_not_found_page_soft_404_heading():
-    html = "<html><body><h1>Page Not Found</h1><p>Sorry</p></body></html>"
+def test_is_not_found_page_soft_404_anywhere_on_page():
+    filler = " ".join(["lorem ipsum"] * 200)
+    html = f"<html><body><main>{filler}</main><footer>Error 404</footer></body></html>"
+    assert is_not_found_page(_html_response(html)) is True
+
+
+def test_is_not_found_page_soft_404_in_meta():
+    html = '<html><head><meta name="description" content="404 page"></head><body>Hi</body></html>'
     assert is_not_found_page(_html_response(html)) is True
 
 
@@ -33,6 +47,32 @@ def test_is_not_found_page_valid_page():
     assert is_not_found_page(_html_response(html)) is False
 
 
+def test_extract_page_text_includes_meta_and_attributes():
+    html = (
+        "<html><head><title>Home</title>"
+        '<meta name="description" content="Welcome home"></head>'
+        '<body><img alt="Hero banner" src="/hero.png"><p>Hello</p></body></html>'
+    )
+    text = extract_page_text(html)
+    assert "Home" in text
+    assert "Welcome home" in text
+    assert "Hero banner" in text
+    assert "Hello" in text
+
+
+def test_page_mentions_404_case_insensitive():
+    assert page_mentions_404("error 404") is True
+    assert page_mentions_404("ERROR 404") is True
+    assert page_mentions_404("all good") is False
+
+
+def test_extract_css_urls_import_and_url():
+    css = '@import url("fonts/inter.css"); body { background: url("/bg.png"); }'
+    urls = extract_css_urls(css, "https://example.com/style.css")
+    assert "https://example.com/fonts/inter.css" in urls
+    assert "https://example.com/bg.png" in urls
+
+
 def test_normalize_url_drops_fragment():
     assert normalize_url("https://example.com/path#section") == "https://example.com/path"
 

web_snapshot_cli-1.3.0/tests/test_wordlist.py

@@ -0,0 +1,92 @@
+from unittest.mock import AsyncMock, MagicMock
+
+import httpx
+import pytest
+
+from snapshot.wordlist import _probe_url, paths_for_word, resolve_wordlist_sources
+
+
+def test_paths_for_word_file_entry():
+    paths = paths_for_word("robots.txt", ())
+    assert paths == ["/robots.txt"]
+
+
+def test_paths_for_word_directory_entry():
+    paths = paths_for_word("admin", (".html",))
+    assert "/admin" in paths
+    assert "/admin/" in paths
+    assert "/admin.html" in paths
+
+
+def test_resolve_wordlist_sources_gobuster_defaults():
+    assert resolve_wordlist_sources(True, []) == ["common", "large"]
+
+
+def test_resolve_wordlist_sources_custom():
+    assert resolve_wordlist_sources(False, ["common", "/tmp/list.txt"]) == [
+        "common",
+        "/tmp/list.txt",
+    ]
+
+
+def test_load_builtin_common():
+    from snapshot.wordlist import load_words
+
+    words = load_words(["common"])
+    assert "robots.txt" in words
+    assert "sitemap.xml" in words
+
+
+@pytest.mark.asyncio
+async def test_probe_url_rejects_soft_404():
+    client = MagicMock(spec=httpx.AsyncClient)
+    response = MagicMock()
+    response.status_code = 200
+    response.headers = {"content-type": "text/html"}
+    response.url = "https://example.com/missing"
+    response.text = (
+        "<html><head><title>404 Not Found</title></head><body><h1>404</h1></body></html>"
+    )
+    client.get = AsyncMock(return_value=response)
+
+    assert await _probe_url(client, "https://example.com/missing") is False
+    client.get.assert_awaited_once_with("https://example.com/missing", follow_redirects=True)
+
+
+@pytest.mark.asyncio
+async def test_probe_url_accepts_valid_page():
+    client = MagicMock(spec=httpx.AsyncClient)
+    response = MagicMock()
+    response.status_code = 200
+    response.headers = {"content-type": "text/html"}
+    response.url = "https://example.com/about"
+    response.text = "<html><head><title>About</title></head><body><h1>About</h1></body></html>"
+    client.get = AsyncMock(return_value=response)
+
+    assert await _probe_url(client, "https://example.com/about") is True
+
+
+@pytest.mark.asyncio
+async def test_probe_url_rejects_hard_404_after_redirect():
+    client = MagicMock(spec=httpx.AsyncClient)
+    response = MagicMock()
+    response.status_code = 404
+    response.headers = {"content-type": "text/html"}
+    response.url = "https://example.com/missing"
+    response.text = "<html><body>missing</body></html>"
+    client.get = AsyncMock(return_value=response)
+
+    assert await _probe_url(client, "https://example.com/missing/") is False
+
+
+@pytest.mark.asyncio
+async def test_probe_url_does_not_count_redirect_without_final_page():
+    client = MagicMock(spec=httpx.AsyncClient)
+    response = MagicMock()
+    response.status_code = 308
+    response.headers = {"content-type": "text/html"}
+    response.url = "https://example.com/old/"
+    response.text = ""
+    client.get = AsyncMock(return_value=response)
+
+    assert await _probe_url(client, "https://example.com/old/") is False

web_snapshot_cli-1.1.0/tests/test_wordlist.py

@@ -1,69 +0,0 @@
-from unittest.mock import AsyncMock, MagicMock
-
-import httpx
-import pytest
-
-from snapshot.wordlist import _probe_url, paths_for_word, resolve_wordlist_sources
-
-
-def test_paths_for_word_file_entry():
-    paths = paths_for_word("robots.txt", ())
-    assert paths == ["/robots.txt"]
-
-
-def test_paths_for_word_directory_entry():
-    paths = paths_for_word("admin", (".html",))
-    assert "/admin" in paths
-    assert "/admin/" in paths
-    assert "/admin.html" in paths
-
-
-def test_resolve_wordlist_sources_gobuster_defaults():
-    assert resolve_wordlist_sources(True, []) == ["common", "large"]
-
-
-def test_resolve_wordlist_sources_custom():
-    assert resolve_wordlist_sources(False, ["common", "/tmp/list.txt"]) == [
-        "common",
-        "/tmp/list.txt",
-    ]
-
-
-def test_load_builtin_common():
-    from snapshot.wordlist import load_words
-
-    words = load_words(["common"])
-    assert "robots.txt" in words
-    assert "sitemap.xml" in words
-
-
-@pytest.mark.asyncio
-async def test_probe_url_rejects_soft_404():
-    client = MagicMock(spec=httpx.AsyncClient)
-    head_response = MagicMock()
-    head_response.status_code = 200
-    get_response = MagicMock()
-    get_response.status_code = 200
-    get_response.headers = {"content-type": "text/html"}
-    get_response.url = "https://example.com/missing"
-    get_response.text = (
-        "<html><head><title>404 Not Found</title></head><body><h1>404</h1></body></html>"
-    )
-    client.request = AsyncMock(side_effect=[head_response, get_response])
-
-    assert await _probe_url(client, "https://example.com/missing") is False
-
-
-@pytest.mark.asyncio
-async def test_probe_url_accepts_valid_page():
-    client = MagicMock(spec=httpx.AsyncClient)
-    head_response = MagicMock()
-    head_response.status_code = 200
-    get_response = MagicMock()
-    get_response.status_code = 200
-    get_response.headers = {"content-type": "text/html"}
-    get_response.url = "https://example.com/about"
-    get_response.text = "<html><head><title>About</title></head><body><h1>About</h1></body></html>"
-    client.request = AsyncMock(side_effect=[head_response, get_response])
-
-    assert await _probe_url(client, "https://example.com/about") is True