pagefetch 0.3.0__tar.gz → 0.4.0__tar.gz

{pagefetch-0.3.0 → pagefetch-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pagefetch
-Version: 0.3.0
+Version: 0.4.0
 Summary: Asynchronous HTTP-first web fetching with conservative content extraction and browser fallback
 Author: PageFetch contributors
 License-Expression: MIT

{pagefetch-0.3.0 → pagefetch-0.4.0}/pagefetch/cli.py

@@ -116,10 +116,17 @@ async def _run(args: argparse.Namespace) -> int:
         proxy=config.proxy,
         cache_enabled=config.cache_enabled,
         cache_ttl=config.cache_ttl,
+        cache_path=config.cache_path,
         http_concurrency=config.http_concurrency,
         browser_concurrency=config.browser_concurrency,
         http_timeout=config.http_timeout,
         browser_timeout=config.browser_timeout,
+        retries_http=config.retries_http,
+        retries_browser=config.retries_browser,
+        max_redirects=config.max_redirects,
+        max_content_size=config.max_content_size,
+        confidence_threshold=config.confidence_threshold,
+        raise_on_error=config.raise_on_error,
     ) as client:
         results = await client.fetch_many(urls)
     rendered = _render(results, args.format, args.include_html)

{pagefetch-0.3.0 → pagefetch-0.4.0}/pagefetch/client.py

@@ -108,7 +108,7 @@ class PageFetch:
                 return self
             if self._closed:
                 raise RuntimeError("PageFetch has already been closed")
-            ensure_runtime_requirements()
+            await asyncio.to_thread(ensure_runtime_requirements)
             if self._cache:
                 try:
                     await self._cache.start()
@@ -210,8 +210,11 @@ class PageFetch:
                 "browser_timeout": self.config.browser_timeout,
                 "confidence_threshold": self.config.confidence_threshold,
                 "headers": BROWSER_HEADERS,
+                "http_timeout": self.config.http_timeout,
                 "max_content_size": self.config.max_content_size,
+                "max_redirects": self.config.max_redirects,
                 "retries_browser": self.config.retries_browser,
+                "retries_http": self.config.retries_http,
             },
         )
         fetch_warnings = list(self._startup_warnings)
@@ -319,8 +322,14 @@ class PageFetch:
                     raise_on_error=_raise_on_error,
                 )
             except PageFetchError as exc:
+                # Use the raw item string — normalize_url would re-raise
+                # ValueError for the same invalid URL that caused the error.
+                try:
+                    result_url = normalize_url(item)
+                except ValueError:
+                    result_url = str(item)
                 return FetchResult(
-                    url=normalize_url(item),
+                    url=result_url,
                     success=False,
                     proxy_provider=proxy or self.config.proxy,
                     error=exc.error,
@@ -330,7 +339,16 @@ class PageFetch:
 
         fetched = await asyncio.gather(*(one(item) for item in unique))
         by_url = dict(zip(unique, fetched, strict=True))
-        return [by_url[item] for item in ordered]
+        # For duplicate URLs, return independent copies so callers
+        # do not accidentally alias mutable fields across entries.
+        from copy import deepcopy
+        results: list[FetchResult] = []
+        for item in ordered:
+            result = by_url[item]
+            if results and results[-1] is result:
+                result = deepcopy(result)
+            results.append(result)
+        return results
 
     async def _fetch_http_or_auto(self, url: str, mode: Literal["auto", "http", "browser"], proxy: str) -> FetchResult:
         try:
@@ -358,6 +376,15 @@ class PageFetch:
             return self._result_from_xml(url, response, proxy)
         if content_type.startswith("text/plain"):
             return self._result_from_text(url, response, proxy)
+        if not self._is_html_like(content_type):
+            raise TransportFailure(
+                FetchErrorInfo(
+                    "unsupported_content_type",
+                    f"Content type {content_type!r} is not HTML; cannot process with HTTP/auto mode",
+                    False,
+                ),
+                status_code=response.status_code,
+            )
         html = self._decode(response)
         raw_soup = BeautifulSoup(html, "lxml")
         report = analyze_html(html)
@@ -616,7 +643,16 @@ class PageFetch:
 
     @staticmethod
     def _is_xml(content_type: str) -> bool:
-        return any(value in content_type for value in XML_TYPES)
+        if content_type in ("application/xml", "text/xml"):
+            return True
+        # Match XML-based subtypes like application/atom+xml, image/svg+xml
+        # but NOT application/xhtml+xml (which is HTML5, not generic XML).
+        return "+xml" in content_type and content_type != "application/xhtml+xml"
+
+    @staticmethod
+    def _is_html_like(content_type: str) -> bool:
+        """Return True when *content_type* should be processed as HTML."""
+        return content_type in ("text/html", "application/xhtml+xml")
 
     @staticmethod
     def _validate_fetch_options(mode: str, proxy: str) -> None:

{pagefetch-0.3.0 → pagefetch-0.4.0}/pagefetch/fetching/http.py

@@ -123,8 +123,28 @@ class HTTPFetcher:
 
     @staticmethod
     async def _backoff(attempt: int, retry_after: str | None = None) -> None:
-        if retry_after and retry_after.isdigit():
-            delay = min(float(retry_after), 10.0)
+        if retry_after:
+            seconds = HTTPFetcher._parse_retry_after(retry_after)
+            if seconds is not None:
+                delay = min(seconds, 10.0)
+            else:
+                delay = min(0.25 * (2**attempt) + random.uniform(0, 0.15), 3.0)
         else:
             delay = min(0.25 * (2**attempt) + random.uniform(0, 0.15), 3.0)
         await asyncio.sleep(delay)
+
+    @staticmethod
+    def _parse_retry_after(value: str) -> float | None:
+        """Parse a Retry-After header as delta-seconds or HTTP-date."""
+        if value.isdigit():
+            return float(value)
+        # Try HTTP-date (RFC 7231), e.g. "Wed, 21 Oct 2015 07:28:00 GMT"
+        try:
+            from email.utils import parsedate_to_datetime
+            from datetime import UTC, datetime
+            retry_dt = parsedate_to_datetime(value)
+            now = datetime.now(UTC)
+            delta = (retry_dt - now).total_seconds()
+            return max(0.0, delta)
+        except (ValueError, TypeError, OverflowError):
+            return None

{pagefetch-0.3.0 → pagefetch-0.4.0}/pagefetch/interactive.py

@@ -66,6 +66,18 @@ def _render_results(results: list[FetchResult], output_format: str, include_html
     return render_results(results, output_format, include_html=include_html)
 
 
+def _apply_debug(settings: dict) -> None:
+    """Configure the pagefetch logger to match the current debug setting."""
+    logger = logging.getLogger("pagefetch")
+    if settings.get("debug"):
+        if not any(isinstance(h, logging.StreamHandler) for h in logger.handlers):
+            handler = logging.StreamHandler()
+            logger.addHandler(handler)
+        logger.setLevel(logging.DEBUG)
+    else:
+        logger.setLevel(logging.WARNING)
+
+
 async def _fetch_url(client: PageFetch, url: str, settings: dict) -> list[FetchResult]:
     """Fetch a single URL."""
     result = await client.fetch(
@@ -98,7 +110,7 @@ async def _fetch_file(client: PageFetch, filepath: str, settings: dict) -> list[
     )
 
 
-def _handle_fetch(client: PageFetch, settings: dict) -> None:
+def _handle_fetch(client: PageFetch, settings: dict, loop: asyncio.AbstractEventLoop) -> None:
     """Interactive fetch flow: URL or file."""
     _clear_screen()
     _banner()
@@ -114,11 +126,12 @@ def _handle_fetch(client: PageFetch, settings: dict) -> None:
     if choice == "3":
         return
 
+    _apply_debug(settings)
     if choice == "2":
         filepath = _prompt("  File path")
         if not filepath:
             return
-        results = asyncio.run(_fetch_file(client, filepath, settings))
+        results = loop.run_until_complete(_fetch_file(client, filepath, settings))
     else:
         url = _prompt("  URL")
         if not url:
@@ -127,7 +140,7 @@ def _handle_fetch(client: PageFetch, settings: dict) -> None:
         if not url.startswith(("http://", "https://")):
             if _confirm(f"  No scheme detected. Use 'https://{url}'?", default=True):
                 url = f"https://{url}"
-        results = asyncio.run(_fetch_url(client, url, settings))
+        results = loop.run_until_complete(_fetch_url(client, url, settings))
 
     if not results:
         return
@@ -244,6 +257,7 @@ def _settings_menu(settings: dict) -> None:
             settings["include_html"] = _confirm("  Include raw HTML in output?", default=settings.get("include_html", False))
         elif choice == "7":
             settings["debug"] = _confirm("  Enable debug logging?", default=settings.get("debug", False))
+            _apply_debug(settings)
         elif choice == "8":
             settings["no_cache"] = _confirm("  Disable cache?", default=settings.get("no_cache", False))
             if settings.get("no_cache"):
@@ -326,56 +340,58 @@ def interactive_main() -> int:
         "config_file": None,
     }
 
-    if settings.get("debug"):
-        handler = logging.StreamHandler()
-        logging.getLogger("pagefetch").addHandler(handler)
-        logging.getLogger("pagefetch").setLevel(logging.DEBUG)
+    _apply_debug(settings)
 
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
     client = _init_client(settings)
 
-    while True:
-        _clear_screen()
-        _banner()
-
-        print("  1. Fetch URL(s)")
-        print("  2. Settings")
-        print("  3. View current config")
-        print("  4. Exit")
-        print()
-
-        choice = _prompt("  Choose", "1")
-
-        try:
-            if choice == "1":
-                _handle_fetch(client, settings)
-                # Re-create client in case settings changed
-                try:
-                    asyncio.run(client.close())
-                except Exception:
-                    pass
-                client = _init_client(settings)
-            elif choice == "2":
-                _settings_menu(settings)
-                # Re-create client with new settings
-                try:
-                    asyncio.run(client.close())
-                except Exception:
-                    pass
-                client = _init_client(settings)
-            elif choice == "3":
-                _view_config(settings)
-            elif choice == "4":
-                print("\n  Goodbye!")
+    try:
+        while True:
+            _clear_screen()
+            _banner()
+
+            print("  1. Fetch URL(s)")
+            print("  2. Settings")
+            print("  3. View current config")
+            print("  4. Exit")
+            print()
+
+            choice = _prompt("  Choose", "1")
+
+            try:
+                if choice == "1":
+                    _handle_fetch(client, settings, loop)
+                    # Re-create client in case settings changed
+                    try:
+                        loop.run_until_complete(client.close())
+                    except Exception:
+                        pass
+                    client = _init_client(settings)
+                elif choice == "2":
+                    _settings_menu(settings)
+                    # Re-create client with new settings
+                    try:
+                        loop.run_until_complete(client.close())
+                    except Exception:
+                        pass
+                    client = _init_client(settings)
+                elif choice == "3":
+                    _view_config(settings)
+                elif choice == "4":
+                    print("\n  Goodbye!")
+                    break
+            except KeyboardInterrupt:
+                print("\n\n  Interrupted. Goodbye!")
                 break
-        except KeyboardInterrupt:
-            print("\n\n  Interrupted. Goodbye!")
-            break
-        except Exception as exc:
-            print(f"\n  Unexpected error: {exc}")
-            input("  Press Enter to continue...")
+            except Exception as exc:
+                print(f"\n  Unexpected error: {exc}")
+                input("  Press Enter to continue...")
 
-    try:
-        asyncio.run(client.close())
-    except Exception:
-        pass
+        try:
+            loop.run_until_complete(client.close())
+        except Exception:
+            pass
+    finally:
+        loop.close()
     return 0

{pagefetch-0.3.0 → pagefetch-0.4.0}/pagefetch/processing/detector.py

@@ -1,164 +1,191 @@
-"""Multi-signal HTML completeness analysis."""
-
-from __future__ import annotations
-
-import math
-import re
-from dataclasses import dataclass
-
-from bs4 import BeautifulSoup
-
-CHALLENGE_PATTERNS = (
-    "checking your browser",
-    "verify you are human",
-    "attention required",
-    "access denied",
-    "unusual traffic",
-    "cf-chl-",
-    "captcha",
-)
-JS_PATTERNS = (
-    "enable javascript",
-    "javascript is required",
-    "you need to enable javascript",
-    "please turn on javascript",
-)
-FRAMEWORK_PATTERNS = (
-    "__next_data__",
-    "__nuxt__",
-    "data-reactroot",
-    "data-v-app",
-    "ng-version",
-    "__svelte",
-    "astro-island",
-    "__remixcontext",
-)
-PLACEHOLDER_PATTERNS = ("loading...", "loading…", "skeleton", "spinner", "please wait")
-WALL_PATTERNS = ("sign in to continue", "log in to continue", "consent required", "accept cookies to continue")
-
-
-@dataclass(slots=True, frozen=True)
-class ConfidenceReport:
-    score: float
-    reasons: tuple[str, ...]
-    challenge: bool = False
-    javascript_shell: bool = False
-
-
-def analyze_html(html: str, *, soup: BeautifulSoup | None = None) -> ConfidenceReport:
-    """Estimate document completeness from several positive and negative signals."""
-    if not html or not html.strip():
-        return ConfidenceReport(0.0, ("empty document",), javascript_shell=True)
-    if soup is None:
-        soup = BeautifulSoup(html, "lxml")
-    title = soup.title.get_text(" ", strip=True) if soup.title else ""
-
-    # Single pass: capture script payload size AND decompose noise tags.
-    script_size = 0
-    for tag in soup(["script", "style", "template", "noscript"]):
-        if tag.name == "script":
-            script_size += len(str(tag))
-        tag.decompose()
-    text = " ".join((soup.body or soup).stripped_strings)
-    lowered_text = text.lower()
-    lowered_html = html.lower()
-    text_len = len(text)
-    word_count = len(re.findall(r"\w+", text, re.UNICODE))
-    reasons: list[str] = []
-    score = 0.20
-
-    # Smooth length contribution: useful up to roughly 1,500 visible characters.
-    score += min(0.38, math.log1p(text_len) / math.log(1501) * 0.38)
-    # Single unified traversal for tag-name-based element collections.
-    paragraphs: list = []
-    headings: list = []
-    semantic: list = []
-    structured: list = []
-    nav_texts: list[str] = []
-    main_texts: list[str] = []
-    for tag in soup.find_all(
-        ["p", "h1", "h2", "h3", "h4", "h5", "h6",
-         "main", "article", "section",
-         "ul", "ol", "table", "pre", "blockquote", "nav"]
-    ):
-        name = tag.name
-        if name == "p":
-            paragraphs.append(tag)
-        elif len(name) == 2 and name[0] == "h" and name[1].isdigit():
-            headings.append(tag)
-        elif name in {"main", "article", "section"}:
-            semantic.append(tag)
-            if name in {"main", "article"}:
-                main_texts.append(tag.get_text(" ", strip=True))
-        elif name in {"ul", "ol", "table", "pre", "blockquote"}:
-            structured.append(tag)
-        elif name == "nav":
-            nav_texts.append(tag.get_text(" ", strip=True))
-    if paragraphs:
-        score += min(0.12, len(paragraphs) * 0.025)
-    if headings:
-        score += min(0.07, len(headings) * 0.02)
-    if semantic and any(len(tag.get_text(" ", strip=True)) > 120 for tag in semantic):
-        score += 0.10
-    if structured:
-        score += min(0.06, len(structured) * 0.015)
-    if word_count > 100 and len(set(text.lower().split())) / max(word_count, 1) > 0.35:
-        score += 0.05
-    if text_len > 500 and len(paragraphs) >= 3:
-        score += 0.05
-    title_tokens = {token for token in re.findall(r"\w+", title.lower()) if len(token) > 3}
-    if title_tokens and title_tokens.intersection(set(re.findall(r"\w+", lowered_text))):
-        score += 0.03
-    if soup.find("meta", attrs={"name": re.compile(r"description", re.I)}):
-        score += 0.02
-
-    challenge = any(pattern in lowered_html for pattern in CHALLENGE_PATTERNS)
-    explicit_js = any(pattern in lowered_text for pattern in JS_PATTERNS)
-    framework = any(pattern in lowered_html for pattern in FRAMEWORK_PATTERNS)
-    mounts = soup.select("#app:empty, #root:empty, #__next:empty, [data-reactroot]:empty")
-    shell = explicit_js or bool(mounts) or (framework and text_len < 250) or (
-        script_size > max(10_000, text_len * 8) and text_len < 400
-    )
-    placeholder = any(pattern in lowered_text for pattern in PLACEHOLDER_PATTERNS)
-    wall = any(pattern in lowered_text for pattern in WALL_PATTERNS)
-    refresh = bool(soup.find("meta", attrs={"http-equiv": re.compile(r"^refresh$", re.I)}))
-    navigation_text = " ".join(nav_texts)
-    main_text = " ".join(main_texts)
-
-    if text_len < 80:
-        score -= 0.30
-        reasons.append("very little visible text")
-    elif text_len < 250:
-        score -= 0.14
-        reasons.append("short visible text")
-    if challenge:
-        score = min(score, 0.08)
-        reasons.append("challenge or anti-bot page detected")
-    if explicit_js:
-        score -= 0.35
-        reasons.append("document asks for JavaScript")
-    if mounts:
-        score -= 0.22
-        reasons.append("empty application mount point")
-    if framework and text_len < 400:
-        score -= 0.15
-        reasons.append("framework bootstrap with little rendered text")
-    if script_size > max(10_000, text_len * 8) and text_len < 400:
-        score -= 0.12
-        reasons.append("large script payload with little visible content")
-    if placeholder and text_len < 400:
-        score -= 0.18
-        shell = True
-        reasons.append("loading placeholder detected")
-    if wall and len(main_text) < 200:
-        score -= 0.20
-        reasons.append("login or consent wall detected")
-    if refresh and text_len < 400:
-        score -= 0.15
-        reasons.append("suspicious redirect document")
-    if text_len and len(navigation_text) / text_len > 0.75 and len(main_text) < 120:
-        score -= 0.12
-        reasons.append("document is mostly navigation")
-    if not reasons:
-        reasons.append("document contains substantial rendered content")
-    return ConfidenceReport(round(max(0.0, min(1.0, score)), 3), tuple(reasons), challenge, shell)
+"""Multi-signal HTML completeness analysis."""
+
+from __future__ import annotations
+
+import math
+import re
+from dataclasses import dataclass
+
+from bs4 import BeautifulSoup
+
+CHALLENGE_PATTERNS = (
+    "checking your browser",
+    "verify you are human",
+    "attention required",
+    "access denied",
+    "unusual traffic",
+    "cf-chl-",
+    "captcha",
+)
+JS_PATTERNS = (
+    "enable javascript",
+    "javascript is required",
+    "you need to enable javascript",
+    "please turn on javascript",
+)
+FRAMEWORK_PATTERNS = (
+    "__next_data__",
+    "__nuxt__",
+    "data-reactroot",
+    "data-v-app",
+    "ng-version",
+    "__svelte",
+    "astro-island",
+    "__remixcontext",
+)
+PLACEHOLDER_PATTERNS = ("loading...", "loading…", "skeleton", "spinner", "please wait")
+WALL_PATTERNS = ("sign in to continue", "log in to continue", "consent required", "accept cookies to continue")
+
+
+@dataclass(slots=True, frozen=True)
+class ConfidenceReport:
+    score: float
+    reasons: tuple[str, ...]
+    challenge: bool = False
+    javascript_shell: bool = False
+
+
+def analyze_html(html: str, *, soup: BeautifulSoup | None = None) -> ConfidenceReport:
+    """Estimate document completeness from several positive and negative signals."""
+    if not html or not html.strip():
+        return ConfidenceReport(0.0, ("empty document",), javascript_shell=True)
+    if soup is None:
+        soup = BeautifulSoup(html, "lxml")
+    title = soup.title.get_text(" ", strip=True) if soup.title else ""
+
+    # Single pass: capture script payload size AND decompose noise tags.
+    script_size = 0
+    for tag in soup(["script", "style", "template", "noscript"]):
+        if tag.name == "script":
+            script_size += len(str(tag))
+        tag.decompose()
+    text = " ".join((soup.body or soup).stripped_strings)
+    lowered_text = text.lower()
+    lowered_html = html.lower()
+    text_len = len(text)
+    word_count = len(re.findall(r"\w+", text, re.UNICODE))
+    reasons: list[str] = []
+    score = 0.20
+
+    # Smooth length contribution: useful up to roughly 1,500 visible characters.
+    score += min(0.38, math.log1p(text_len) / math.log(1501) * 0.38)
+    # Single unified traversal for tag-name-based element collections.
+    paragraphs: list = []
+    headings: list = []
+    semantic: list = []
+    structured: list = []
+    nav_texts: list[str] = []
+    main_texts: list[str] = []
+    for tag in soup.find_all(
+        ["p", "h1", "h2", "h3", "h4", "h5", "h6",
+         "main", "article", "section",
+         "ul", "ol", "table", "pre", "blockquote", "nav"]
+    ):
+        name = tag.name
+        if name == "p":
+            paragraphs.append(tag)
+        elif len(name) == 2 and name[0] == "h" and name[1].isdigit():
+            headings.append(tag)
+        elif name in {"main", "article", "section"}:
+            semantic.append(tag)
+            if name in {"main", "article"}:
+                main_texts.append(tag.get_text(" ", strip=True))
+        elif name in {"ul", "ol", "table", "pre", "blockquote"}:
+            structured.append(tag)
+        elif name == "nav":
+            nav_texts.append(tag.get_text(" ", strip=True))
+    if paragraphs:
+        score += min(0.12, len(paragraphs) * 0.025)
+    if headings:
+        score += min(0.07, len(headings) * 0.02)
+    if semantic and any(len(tag.get_text(" ", strip=True)) > 120 for tag in semantic):
+        score += 0.10
+    if structured:
+        score += min(0.06, len(structured) * 0.015)
+    if word_count > 100 and len(set(text.lower().split())) / max(word_count, 1) > 0.35:
+        score += 0.05
+    if text_len > 500 and len(paragraphs) >= 3:
+        score += 0.05
+    title_tokens = {token for token in re.findall(r"\w+", title.lower()) if len(token) > 3}
+    body_tokens = set(re.findall(r"\w+", lowered_text))
+    title_overlap = bool(title_tokens and title_tokens.intersection(body_tokens))
+    if title_overlap:
+        score += 0.03
+    if soup.find("meta", attrs={"name": re.compile(r"description", re.I)}):
+        score += 0.02
+
+    challenge = any(pattern in lowered_html for pattern in CHALLENGE_PATTERNS)
+    explicit_js = any(pattern in lowered_text for pattern in JS_PATTERNS)
+    framework = any(pattern in lowered_html for pattern in FRAMEWORK_PATTERNS)
+    mounts = soup.select("#app:empty, #root:empty, #__next:empty, [data-reactroot]:empty")
+    shell = explicit_js or bool(mounts) or (framework and text_len < 250) or (
+        script_size > max(10_000, text_len * 8) and text_len < 400
+    )
+    placeholder = any(pattern in lowered_text for pattern in PLACEHOLDER_PATTERNS)
+    wall = any(pattern in lowered_text for pattern in WALL_PATTERNS)
+    refresh = bool(soup.find("meta", attrs={"http-equiv": re.compile(r"^refresh$", re.I)}))
+    navigation_text = " ".join(nav_texts)
+    main_text = " ".join(main_texts)
+
+    # Complete short static pages (e.g. example.com) are fully server-rendered and
+    # should not force an expensive browser fallback just because they are brief.
+    coherent_short = (
+        not shell
+        and not challenge
+        and not placeholder
+        and not wall
+        and not refresh
+        and not mounts
+        and not explicit_js
+        and script_size < 2_000
+        and 80 <= text_len < 400
+        and word_count >= 12
+        and (bool(paragraphs) or bool(headings))
+        and (title_overlap or bool(headings))
+    )
+    if coherent_short:
+        score += 0.28
+        if headings and paragraphs:
+            score += 0.08
+        if title_overlap:
+            score += 0.05
+
+    if text_len < 80:
+        score -= 0.30
+        reasons.append("very little visible text")
+    elif text_len < 250 and not coherent_short:
+        score -= 0.14
+        reasons.append("short visible text")
+    elif coherent_short and text_len < 250:
+        reasons.append("short but complete static document")
+    if challenge:
+        score = min(score, 0.08)
+        reasons.append("challenge or anti-bot page detected")
+    if explicit_js:
+        score -= 0.35
+        reasons.append("document asks for JavaScript")
+    if mounts:
+        score -= 0.22
+        reasons.append("empty application mount point")
+    if framework and text_len < 400:
+        score -= 0.15
+        reasons.append("framework bootstrap with little rendered text")
+    if script_size > max(10_000, text_len * 8) and text_len < 400:
+        score -= 0.12
+        reasons.append("large script payload with little visible content")
+    if placeholder and text_len < 400:
+        score -= 0.18
+        shell = True
+        reasons.append("loading placeholder detected")
+    if wall and len(main_text) < 200:
+        score -= 0.20
+        reasons.append("login or consent wall detected")
+    if refresh and text_len < 400:
+        score -= 0.15
+        reasons.append("suspicious redirect document")
+    if text_len and len(navigation_text) / text_len > 0.75 and len(main_text) < 120:
+        score -= 0.12
+        reasons.append("document is mostly navigation")
+    if not reasons:
+        reasons.append("document contains substantial rendered content")
+    return ConfidenceReport(round(max(0.0, min(1.0, score)), 3), tuple(reasons), challenge, shell)

{pagefetch-0.3.0 → pagefetch-0.4.0}/pagefetch/processing/html.py

@@ -38,7 +38,7 @@ def process_html(
     """Extract a high-fidelity structured representation from HTML."""
     raw_soup = soup or BeautifulSoup(html, "lxml")
     title, metadata, warnings = extract_metadata(raw_soup, base_url, response_headers)
-    confidence = analyze_html(html, soup=raw_soup)
+    confidence = analyze_html(html)
     cleaned = clean_html(raw_soup)
     links = extract_links(cleaned, base_url)
     images = extract_images(cleaned, base_url)

{pagefetch-0.3.0 → pagefetch-0.4.0}/pagefetch/utils/durations.py

@@ -15,7 +15,7 @@ def parse_duration(value: str | int | float) -> int:
     if isinstance(value, (int, float)):
         if value < 0:
             raise ValueError("duration cannot be negative")
-        return int(value)
+        return round(value)
     if not isinstance(value, str):
         raise TypeError("duration must be a number or string")
     match = _DURATION_RE.fullmatch(value.strip())

{pagefetch-0.3.0 → pagefetch-0.4.0}/pagefetch.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pagefetch
-Version: 0.3.0
+Version: 0.4.0
 Summary: Asynchronous HTTP-first web fetching with conservative content extraction and browser fallback
 Author: PageFetch contributors
 License-Expression: MIT

{pagefetch-0.3.0 → pagefetch-0.4.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "pagefetch"
-version = "0.3.0"
+version = "0.4.0"
 description = "Asynchronous HTTP-first web fetching with conservative content extraction and browser fallback"
 readme = "README.md"
 requires-python = ">=3.11"

{pagefetch-0.3.0 → pagefetch-0.4.0}/tests/test_processing.py

@@ -47,6 +47,17 @@ def test_processing_preserves_content_and_converts_markdown():
     assert result.images[0].url == "https://example.com/hero.jpg"
 
 
+EXAMPLE_DOMAIN_HTML = (
+    '<!doctype html><html lang="en"><head><title>Example Domain</title>'
+    '<meta name="viewport" content="width=device-width, initial-scale=1">'
+    "<style>body{background:#eee}</style></head><body><div><h1>Example Domain</h1>"
+    "<p>This domain is for use in documentation examples without needing permission. "
+    "Avoid use in operations.</p>"
+    '<p><a href="https://iana.org/domains/example">Learn more</a></p>'
+    "</div></body></html>"
+)
+
+
 def test_detector_distinguishes_rich_content_spa_and_challenge():
     rich = analyze_html(RICH_HTML)
     spa = analyze_html("<html><body><div id='root'></div><script>" + "x" * 12000 + "</script></body></html>")
@@ -56,6 +67,18 @@ def test_detector_distinguishes_rich_content_spa_and_challenge():
     assert challenge.score <= 0.08 and challenge.challenge
 
 
+def test_detector_scores_short_static_pages_highly():
+    """Simple complete pages like example.com must clear the HTTP confidence threshold."""
+    report = analyze_html(EXAMPLE_DOMAIN_HTML)
+    assert report.score >= 0.80
+    assert not report.javascript_shell
+    assert not report.challenge
+    assert "short but complete static document" in report.reasons
+
+    thin = analyze_html("<html><body><p>Small but usable notice.</p></body></html>")
+    assert thin.score < 0.50
+
+
 def test_external_link_is_identified():
     result = process_html('<html><body><a href="https://other.test/x">Other</a></body></html>', "https://example.com/")
     assert result.links[0].internal is False