web-snapshot-cli 1.1.0__tar.gz → 1.2.0__tar.gz

{web_snapshot_cli-1.1.0 → web_snapshot_cli-1.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: web-snapshot-cli
-Version: 1.1.0
+Version: 1.2.0
 Summary: Take offline snapshots of any website and restore them locally
 Project-URL: Homepage, https://github.com/codingsushi79/Snapshot
 Project-URL: Repository, https://github.com/codingsushi79/Snapshot

{web_snapshot_cli-1.1.0 → web_snapshot_cli-1.2.0}/docs/index.html

@@ -50,7 +50,7 @@
         </label>
         <label class="check-item">
           <input type="checkbox" id="gobuster" />
-          <div>--gobuster<span>Bundled wordlists; soft 404 pages are filtered out</span></div>
+          <div>--gobuster<span>Wordlist discovery; skips any page mentioning 404</span></div>
         </label>
         <label class="check-item">
           <input type="checkbox" id="sitemap" />
@@ -65,7 +65,7 @@
       <div class="field snapshot-only">
         <label for="wordlists">Extra wordlists (one per line)</label>
         <textarea id="wordlists" placeholder="common&#10;large&#10;/path/to/SecLists/Discovery/Web-Content/common.txt"></textarea>
-        <div class="hint">Builtin names: <code>common</code>, <code>large</code>. --gobuster uses both by default. Pages that return 200 but show 404/not-found content are skipped.</div>
+        <div class="hint">Builtin names: <code>common</code>, <code>large</code>. --gobuster uses both by default. Any page whose full text contains <code>404</code> is skipped, even on HTTP 200.</div>
       </div>
       <div class="field snapshot-only">
         <label for="wordlistExt">Wordlist extensions</label>
@@ -143,7 +143,7 @@
         </label>
         <label class="check-item">
           <input type="checkbox" id="noAssets" />
-          <div>--no-assets<span>Skip CSS/JS/images</span></div>
+          <div>--no-assets<span>Skip CSS/JS/images/animations</span></div>
         </label>
         <label class="check-item">
           <input type="checkbox" id="verbose" />

{web_snapshot_cli-1.1.0 → web_snapshot_cli-1.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "web-snapshot-cli"
-version = "1.1.0"
+version = "1.2.0"
 description = "Take offline snapshots of any website and restore them locally"
 readme = "README.md"
 license = "MIT"

{web_snapshot_cli-1.1.0 → web_snapshot_cli-1.2.0}/src/snapshot/__init__.py

@@ -1,3 +1,3 @@
 """snapshot — offline website snapshots."""
 
-__version__ = "1.1.0"
+__version__ = "1.2.0"

{web_snapshot_cli-1.1.0 → web_snapshot_cli-1.2.0}/src/snapshot/downloader.py

@@ -20,6 +20,7 @@ from snapshot.manifest import MANIFEST_NAME, SnapshotManifest
 from snapshot.utils import (
     SRCSET_ATTRS,
     URL_ATTRS,
+    extract_css_urls,
     is_not_found_page,
     normalize_url,
     page_local_path,
@@ -419,12 +420,15 @@ class SnapshotEngine:
                         absolute = normalize_url(piece, page_url)
                         if absolute and not self._is_page_url(absolute):
                             urls.add(absolute)
-        for style in soup.find_all("style"):
-            if style.string:
-                for match in re.findall(r"url\(([^)]+)\)", style.string):
+            style = tag.get("style")
+            if style:
+                for match in re.findall(r"url\(([^)]+)\)", style):
                     absolute = normalize_url(match.strip("'\""), page_url)
                     if absolute and not self._is_page_url(absolute):
                         urls.add(absolute)
+        for style in soup.find_all("style"):
+            if style.string:
+                urls.update(extract_css_urls(style.string, page_url))
         return urls
 
     def _collect_page_links(self, soup: BeautifulSoup, page_url: str) -> list[str]:
@@ -466,7 +470,7 @@ class SnapshotEngine:
                     lambda asset_url: self._css_mapper(asset_url, local_path),
                 )
                 local_path.write_text(css, encoding="utf-8")
-                nested = self._extract_css_urls(css, url)
+                nested = extract_css_urls(css, url)
                 await asyncio.gather(*(self._download_asset(u) for u in nested))
             else:
                 local_path.write_bytes(response.content)
@@ -480,14 +484,6 @@ class SnapshotEngine:
             self._url_to_path[asset_url] = target
         return relative_href(css_path, target)
 
-    def _extract_css_urls(self, css: str, base_url: str) -> set[str]:
-        urls: set[str] = set()
-        for match in re.findall(r"url\(([^)]+)\)", css):
-            absolute = normalize_url(match.strip("'\""), base_url)
-            if absolute:
-                urls.add(absolute)
-        return urls
-
     def _rewrite_html(self, soup: BeautifulSoup, page_url: str, page_path: Path) -> None:
         for tag in soup.find_all(True):
             for attr in URL_ATTRS:
@@ -533,6 +529,8 @@ class SnapshotEngine:
     def _render_page(self, soup: BeautifulSoup) -> str:
         if self.options.lang == "md":
             return self._html_to_markdown(soup)
+        if not soup.find("html"):
+            return f"<!DOCTYPE html>\n{str(soup)}"
         return str(soup)
 
     def _html_to_markdown(self, soup: BeautifulSoup) -> str:

{web_snapshot_cli-1.1.0 → web_snapshot_cli-1.2.0}/src/snapshot/utils.py

@@ -12,8 +12,28 @@ if TYPE_CHECKING:
     import httpx
 
 # Attributes that may contain fetchable URLs.
-URL_ATTRS = ("href", "src", "poster", "data-src", "data-background")
+URL_ATTRS = (
+    "href",
+    "src",
+    "poster",
+    "data-src",
+    "data-background",
+    "data-lazy-src",
+    "data-original",
+    "data-lazy",
+    "data-bg",
+    "data-background-image",
+    "data-image",
+    "data-href",
+    "data-url",
+    "data-video",
+    "data-poster",
+    "data-anim-src",
+    "data-animation",
+)
 SRCSET_ATTRS = ("srcset", "data-srcset")
+TEXT_ATTRS = ("aria-label", "alt", "title", "placeholder")
+_SKIP_TAGS = frozenset({"script", "style", "noscript"})
 
 # Tags treated as HTML pages when crawled.
 PAGE_EXTENSIONS = {".html", ".htm", ".xhtml", ""}
@@ -43,14 +63,44 @@ ASSET_EXTENSIONS = {
 
 SKIP_SCHEMES = {"mailto", "tel", "javascript", "data", "blob", "about", "file"}
 
-_NOT_FOUND_TEXT = re.compile(
-    r"\b404\b|not\s+found|page\s+not\s+found|page\s+cannot\s+be\s+found",
-    re.IGNORECASE,
-)
+_404_MARKER = re.compile(r"404", re.IGNORECASE)
+
+
+def extract_page_text(html: str) -> str:
+    """Collect visible and semantic text from an HTML document."""
+    from bs4 import BeautifulSoup
+
+    soup = BeautifulSoup(html, "html.parser")
+    parts: list[str] = []
+
+    if soup.title:
+        parts.append(soup.title.get_text(" ", strip=True))
+
+    for meta in soup.find_all("meta"):
+        content = meta.get("content")
+        if content:
+            parts.append(str(content))
+
+    for tag in soup.find_all(True):
+        for attr in TEXT_ATTRS:
+            value = tag.get(attr)
+            if value:
+                parts.append(str(value))
+
+    for skip_tag in list(soup.find_all(_SKIP_TAGS)):
+        skip_tag.decompose()
+
+    parts.append(soup.get_text(" ", strip=True))
+    return " ".join(part for part in parts if part)
+
+
+def page_mentions_404(text: str) -> bool:
+    """Return True when page text contains a 404 marker."""
+    return bool(_404_MARKER.search(text))
 
 
 def is_not_found_page(response: httpx.Response) -> bool:
-    """Detect hard and soft 404 pages (HTTP 404 or HTML error content with 200 status)."""
+    """Detect hard and soft 404 pages (HTTP 404 or any page text mentioning 404)."""
     if response.status_code == 404:
         return True
     if response.status_code not in {200, 201, 204}:
@@ -62,32 +112,16 @@ def is_not_found_page(response: httpx.Response) -> bool:
         return False
 
     try:
-        text = response.text
+        html = response.text
     except Exception:  # noqa: BLE001
         return False
 
-    from bs4 import BeautifulSoup
-
-    soup = BeautifulSoup(text, "html.parser")
-
-    title = soup.title.get_text(" ", strip=True) if soup.title else ""
-    if title and _NOT_FOUND_TEXT.search(title):
+    if page_mentions_404(extract_page_text(html)):
         return True
 
-    for tag_name in ("h1", "h2"):
-        heading = soup.find(tag_name)
-        if heading:
-            heading_text = heading.get_text(" ", strip=True)
-            if heading_text and _NOT_FOUND_TEXT.search(heading_text):
-                return True
-
-    body = soup.body
-    if body:
-        body_text = body.get_text(" ", strip=True)
-        if body_text and len(body_text) < 500 and _NOT_FOUND_TEXT.search(body_text):
-            return True
-
-    return False
+    stripped = re.sub(r"(?is)<script[^>]*>.*?</script>", " ", html)
+    stripped = re.sub(r"(?is)<style[^>]*>.*?</style>", " ", stripped)
+    return page_mentions_404(stripped)
 
 
 def normalize_url(url: str, base: str | None = None) -> str | None:
@@ -187,6 +221,20 @@ def parse_srcset(value: str) -> list[str]:
     return urls
 
 
+def extract_css_urls(css: str, base_url: str) -> set[str]:
+    """Extract linked asset URLs from CSS, including @import rules."""
+    urls: set[str] = set()
+    for match in re.findall(r"url\(([^)]+)\)", css):
+        absolute = normalize_url(match.strip("'\""), base_url)
+        if absolute:
+            urls.add(absolute)
+    for match in re.findall(r"@import\s+(?:url\()?['\"]?([^'\")\s;]+)", css):
+        absolute = normalize_url(match.strip("'\""), base_url)
+        if absolute:
+            urls.add(absolute)
+    return urls
+
+
 def rewrite_css_urls(css: str, base_url: str, mapper) -> str:
     """Rewrite url(...) references in CSS using mapper(url) -> local path or None."""
 

{web_snapshot_cli-1.1.0 → web_snapshot_cli-1.2.0}/tests/test_utils.py

@@ -1,7 +1,15 @@
 from pathlib import Path
 from unittest.mock import MagicMock
 
-from snapshot.utils import is_not_found_page, normalize_url, page_local_path, url_to_local_path
+from snapshot.utils import (
+    extract_css_urls,
+    extract_page_text,
+    is_not_found_page,
+    normalize_url,
+    page_local_path,
+    page_mentions_404,
+    url_to_local_path,
+)
 
 
 def _html_response(html: str, status_code: int = 200) -> MagicMock:
@@ -23,8 +31,14 @@ def test_is_not_found_page_soft_404_title():
     assert is_not_found_page(_html_response(html)) is True
 
 
-def test_is_not_found_page_soft_404_heading():
-    html = "<html><body><h1>Page Not Found</h1><p>Sorry</p></body></html>"
+def test_is_not_found_page_soft_404_anywhere_on_page():
+    filler = " ".join(["lorem ipsum"] * 200)
+    html = f"<html><body><main>{filler}</main><footer>Error 404</footer></body></html>"
+    assert is_not_found_page(_html_response(html)) is True
+
+
+def test_is_not_found_page_soft_404_in_meta():
+    html = '<html><head><meta name="description" content="404 page"></head><body>Hi</body></html>'
     assert is_not_found_page(_html_response(html)) is True
 
 
@@ -33,6 +47,32 @@ def test_is_not_found_page_valid_page():
     assert is_not_found_page(_html_response(html)) is False
 
 
+def test_extract_page_text_includes_meta_and_attributes():
+    html = (
+        "<html><head><title>Home</title>"
+        '<meta name="description" content="Welcome home"></head>'
+        '<body><img alt="Hero banner" src="/hero.png"><p>Hello</p></body></html>'
+    )
+    text = extract_page_text(html)
+    assert "Home" in text
+    assert "Welcome home" in text
+    assert "Hero banner" in text
+    assert "Hello" in text
+
+
+def test_page_mentions_404_case_insensitive():
+    assert page_mentions_404("error 404") is True
+    assert page_mentions_404("ERROR 404") is True
+    assert page_mentions_404("all good") is False
+
+
+def test_extract_css_urls_import_and_url():
+    css = '@import url("fonts/inter.css"); body { background: url("/bg.png"); }'
+    urls = extract_css_urls(css, "https://example.com/style.css")
+    assert "https://example.com/fonts/inter.css" in urls
+    assert "https://example.com/bg.png" in urls
+
+
 def test_normalize_url_drops_fragment():
     assert normalize_url("https://example.com/path#section") == "https://example.com/path"