pagefetch 0.3.0__tar.gz → 0.5.0__tar.gz

{pagefetch-0.3.0 → pagefetch-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pagefetch
-Version: 0.3.0
+Version: 0.5.0
 Summary: Asynchronous HTTP-first web fetching with conservative content extraction and browser fallback
 Author: PageFetch contributors
 License-Expression: MIT
@@ -142,9 +142,10 @@ async with PageFetch(mode="auto") as client:
 4. **Content analysis** — the `confidence` score evaluates HTML completeness using  
    text density, structural markup, heading presence, link counts, and common blocking signals  
    (captcha walls, empty bodies, access-denied patterns).
-5. **Browser fallback** (auto mode only) — if confidence is below the threshold (default 0.80),  
-   Camoufox takes over: renders JavaScript, scrolls for lazy-loaded content, waits for  
-   network stability, and re-extracts.
+5. **Browser fallback** (auto mode only) — Camoufox takes over only when HTTP content  
+   confidence is below the threshold (default 0.80), or when the server returns a blocked  
+   status (403/429). Timeouts, connection failures, 404s, and 5xx responses fail fast at  
+   the HTTP layer instead of waiting on a browser navigation.
 6. **Processing pipeline** — cleaned HTML → extracted links, images, metadata →  
    converted to Markdown via a custom converter that preserves tables, code blocks,  
    and nested lists.
@@ -216,6 +217,7 @@ client = PageFetch(
     max_redirects=10,         # Maximum redirect chain
     max_content_size=25 * 1024 * 1024,  # Max response body bytes (25 MiB)
     confidence_threshold=0.80,    # Min confidence before browser fallback
+    block_images=True,         # Block image loading in browser mode to save bandwidth
     raise_on_error=False,     # Raise PageFetchError instead of returning error result
 )
 ```
@@ -264,7 +266,7 @@ class FetchResult:
     text: str | None            # Plain-text body fallback
     metadata: dict              # OpenGraph, Twitter Cards, meta tags
     links: list[LinkInfo]       # All <a> tags with text, URL, rel
-    images: list[ImageInfo]     # All <img> tags with src, alt, title
+    images: list[ImageInfo]     # All <img> tags with url, alt, title
     fetch_method: str | None    # "http" or "browser"
     proxy_provider: str         # "none", "decodo", or "dataimpulse"
     content_confidence: float | None  # 0–1 completeness score (None for browser mode)
@@ -300,6 +302,9 @@ PageFetch ships with a command-line interface accessible via `pagefetch`:
 # Fetch and print Markdown
 pagefetch https://example.com --format markdown
 
+# Fetch and print raw HTML
+pagefetch https://example.com --format html
+
 # Fetch from a list and output JSON
 pagefetch urls.txt --format json --mode auto
 
@@ -311,10 +316,18 @@ pagefetch https://example.com --format json --include-html
 
 # Multiple URLs from a file (one URL per line)
 pagefetch urls.txt --format json --mode auto
+
+# Load configuration from a YAML file with CLI overrides
+pagefetch --config config.yaml --mode browser https://example.com
+
+# Override cache TTL and disable image loading
+pagefetch https://example.com --cache-ttl 1h --block-images
 ```
 
 CLI arguments map directly to the Python API — `--mode`, `--proxy`, `--timeout`,
-`--http-concurrency`, and `--browser-concurrency` are all supported.
+`--browser-timeout`, `--http-concurrency`, `--browser-concurrency`, `--cache-ttl`,
+`--no-cache`, `--block-images` / `--no-block-images`, `--include-html`,
+`--debug`, and `--config` are all supported.
 
 ---
 
@@ -396,6 +409,7 @@ at the HTTP response level before any processing pipeline runs.
 | `max_redirects` | `int` | `10` | Maximum redirect chain to follow |
 | `max_content_size` | `int` | `25 MiB` | Maximum response body in bytes |
 | `confidence_threshold` | `float` | `0.80` | Threshold for browser fallback in auto mode |
+| `block_images` | `bool` | `True` | Block image loading in browser mode to save bandwidth |
 | `raise_on_error` | `bool` | `False` | Raise `PageFetchError` on failure instead of returning error result |
 
 ---

{pagefetch-0.3.0 → pagefetch-0.5.0}/README.md

@@ -104,9 +104,10 @@ async with PageFetch(mode="auto") as client:
 4. **Content analysis** — the `confidence` score evaluates HTML completeness using  
    text density, structural markup, heading presence, link counts, and common blocking signals  
    (captcha walls, empty bodies, access-denied patterns).
-5. **Browser fallback** (auto mode only) — if confidence is below the threshold (default 0.80),  
-   Camoufox takes over: renders JavaScript, scrolls for lazy-loaded content, waits for  
-   network stability, and re-extracts.
+5. **Browser fallback** (auto mode only) — Camoufox takes over only when HTTP content  
+   confidence is below the threshold (default 0.80), or when the server returns a blocked  
+   status (403/429). Timeouts, connection failures, 404s, and 5xx responses fail fast at  
+   the HTTP layer instead of waiting on a browser navigation.
 6. **Processing pipeline** — cleaned HTML → extracted links, images, metadata →  
    converted to Markdown via a custom converter that preserves tables, code blocks,  
    and nested lists.
@@ -178,6 +179,7 @@ client = PageFetch(
     max_redirects=10,         # Maximum redirect chain
     max_content_size=25 * 1024 * 1024,  # Max response body bytes (25 MiB)
     confidence_threshold=0.80,    # Min confidence before browser fallback
+    block_images=True,         # Block image loading in browser mode to save bandwidth
     raise_on_error=False,     # Raise PageFetchError instead of returning error result
 )
 ```
@@ -226,7 +228,7 @@ class FetchResult:
     text: str | None            # Plain-text body fallback
     metadata: dict              # OpenGraph, Twitter Cards, meta tags
     links: list[LinkInfo]       # All <a> tags with text, URL, rel
-    images: list[ImageInfo]     # All <img> tags with src, alt, title
+    images: list[ImageInfo]     # All <img> tags with url, alt, title
     fetch_method: str | None    # "http" or "browser"
     proxy_provider: str         # "none", "decodo", or "dataimpulse"
     content_confidence: float | None  # 0–1 completeness score (None for browser mode)
@@ -262,6 +264,9 @@ PageFetch ships with a command-line interface accessible via `pagefetch`:
 # Fetch and print Markdown
 pagefetch https://example.com --format markdown
 
+# Fetch and print raw HTML
+pagefetch https://example.com --format html
+
 # Fetch from a list and output JSON
 pagefetch urls.txt --format json --mode auto
 
@@ -273,10 +278,18 @@ pagefetch https://example.com --format json --include-html
 
 # Multiple URLs from a file (one URL per line)
 pagefetch urls.txt --format json --mode auto
+
+# Load configuration from a YAML file with CLI overrides
+pagefetch --config config.yaml --mode browser https://example.com
+
+# Override cache TTL and disable image loading
+pagefetch https://example.com --cache-ttl 1h --block-images
 ```
 
 CLI arguments map directly to the Python API — `--mode`, `--proxy`, `--timeout`,
-`--http-concurrency`, and `--browser-concurrency` are all supported.
+`--browser-timeout`, `--http-concurrency`, `--browser-concurrency`, `--cache-ttl`,
+`--no-cache`, `--block-images` / `--no-block-images`, `--include-html`,
+`--debug`, and `--config` are all supported.
 
 ---
 
@@ -358,6 +371,7 @@ at the HTTP response level before any processing pipeline runs.
 | `max_redirects` | `int` | `10` | Maximum redirect chain to follow |
 | `max_content_size` | `int` | `25 MiB` | Maximum response body in bytes |
 | `confidence_threshold` | `float` | `0.80` | Threshold for browser fallback in auto mode |
+| `block_images` | `bool` | `True` | Block image loading in browser mode to save bandwidth |
 | `raise_on_error` | `bool` | `False` | Raise `PageFetchError` on failure instead of returning error result |
 
 ---

{pagefetch-0.3.0 → pagefetch-0.5.0}/pagefetch/__init__.py

@@ -46,5 +46,3 @@ __all__ = [
     "VALID_PROXIES",
     "ensure_runtime_requirements",
 ]
-
-__version__ = "0.1.0"

{pagefetch-0.3.0 → pagefetch-0.5.0}/pagefetch/bootstrap.py

@@ -24,6 +24,7 @@ _RUNTIME_REQUIREMENTS = (
 )
 _FALSE_VALUES = frozenset({"0", "false", "no", "off"})
 _bootstrap_complete = False
+_camoufox_installed = False
 
 
 class RuntimeBootstrapError(RuntimeError):
@@ -81,15 +82,20 @@ def install_python_requirements() -> None:
 
 def install_camoufox_browser() -> None:
     """Download or update the Camoufox browser binary when it is unavailable."""
+    global _camoufox_installed
+    if _camoufox_installed:
+        return
     try:
         from camoufox.pkgman import camoufox_path
 
         try:
             camoufox_path(download_if_missing=False)
+            _camoufox_installed = True
             return
         except Exception:
             print("[pagefetch] Installing the Camoufox browser runtime...", file=sys.stderr)
         camoufox_path(download_if_missing=True)
+        _camoufox_installed = True
     except Exception as exc:
         raise RuntimeBootstrapError(
             "PageFetch could not install the Camoufox browser runtime. "
@@ -98,17 +104,23 @@ def install_camoufox_browser() -> None:
         ) from exc
 
 
-def ensure_runtime_requirements() -> None:
+def ensure_runtime_requirements(*, needs_browser: bool = True) -> None:
     """Install all runtime requirements once at process startup.
 
     The caller (PageFetch.start) is responsible for ensuring this is
     called at most once per client lifecycle via its own asyncio.Lock.
+
+    *needs_browser* gates Camoufox installation so that HTTP-only
+    clients do not force an unnecessary browser download.
     """
     global _bootstrap_complete
     if not auto_install_enabled():
         return
     if _bootstrap_complete:
+        if needs_browser:
+            install_camoufox_browser()
         return
     install_python_requirements()
-    install_camoufox_browser()
+    if needs_browser:
+        install_camoufox_browser()
     _bootstrap_complete = True

{pagefetch-0.3.0 → pagefetch-0.5.0}/pagefetch/cli.py

@@ -36,6 +36,8 @@ def build_parser() -> argparse.ArgumentParser:
     parser.add_argument("--browser-concurrency", type=int, default=argparse.SUPPRESS)
     parser.add_argument("--timeout", type=float, default=argparse.SUPPRESS)
     parser.add_argument("--browser-timeout", type=float, default=argparse.SUPPRESS)
+    parser.add_argument("--block-images", action="store_true", default=argparse.SUPPRESS, dest="block_images")
+    parser.add_argument("--no-block-images", action="store_false", default=argparse.SUPPRESS, dest="block_images")
     parser.add_argument("--debug", action="store_true")
     return parser
 
@@ -82,6 +84,8 @@ def _build_config(args: argparse.Namespace) -> PageFetchConfig:
         overrides["http_timeout"] = args.timeout
     if hasattr(args, "browser_timeout"):
         overrides["browser_timeout"] = args.browser_timeout
+    if hasattr(args, "block_images"):
+        overrides["block_images"] = args.block_images
 
     if not overrides and not args.no_cache:
         return config
@@ -102,6 +106,7 @@ def _build_config(args: argparse.Namespace) -> PageFetchConfig:
         max_redirects=config.max_redirects,
         max_content_size=config.max_content_size,
         confidence_threshold=config.confidence_threshold,
+        block_images=bool(overrides.get("block_images", config.block_images)),
         raise_on_error=config.raise_on_error,
     )
 
@@ -116,10 +121,18 @@ async def _run(args: argparse.Namespace) -> int:
         proxy=config.proxy,
         cache_enabled=config.cache_enabled,
         cache_ttl=config.cache_ttl,
+        cache_path=config.cache_path,
         http_concurrency=config.http_concurrency,
         browser_concurrency=config.browser_concurrency,
         http_timeout=config.http_timeout,
         browser_timeout=config.browser_timeout,
+        retries_http=config.retries_http,
+        retries_browser=config.retries_browser,
+        max_redirects=config.max_redirects,
+        max_content_size=config.max_content_size,
+        confidence_threshold=config.confidence_threshold,
+        block_images=config.block_images,
+        raise_on_error=config.raise_on_error,
     ) as client:
         results = await client.fetch_many(urls)
     rendered = _render(results, args.format, args.include_html)

{pagefetch-0.3.0 → pagefetch-0.5.0}/pagefetch/client.py

@@ -27,7 +27,7 @@ from .constants import (
 from .exceptions import PageFetchError
 from .fetching import BrowserFetcher, HTTPFetcher, HTTPResponse, TransportFailure
 from .models import FetchErrorInfo, FetchResult
-from .processing.detector import analyze_html
+from .processing.detector import ConfidenceReport, analyze_html
 from .processing.html import process_html
 from .processing.non_html import process_pdf, process_text, process_xml
 from .proxy import ProxyConfigurationError, resolve_proxy
@@ -62,6 +62,7 @@ class PageFetch:
         max_redirects: int = 10,
         max_content_size: int = 25 * 1024 * 1024,
         confidence_threshold: float = 0.80,
+        block_images: bool = True,
         raise_on_error: bool = False,
     ) -> None:
         self.config = PageFetchConfig.build(
@@ -79,6 +80,7 @@ class PageFetch:
             max_redirects=max_redirects,
             max_content_size=max_content_size,
             confidence_threshold=confidence_threshold,
+            block_images=block_images,
             raise_on_error=raise_on_error,
         )
         self._http_semaphore = asyncio.Semaphore(self.config.http_concurrency)
@@ -93,6 +95,8 @@ class PageFetch:
         self._closed = False
         self._lifecycle_lock = asyncio.Lock()
         self._startup_warnings: list[str] = []
+        self._active_fetches = 0
+        self._active_fetches_lock = asyncio.Lock()
 
     async def __aenter__(self) -> PageFetch:
         await self.start()
@@ -108,7 +112,10 @@ class PageFetch:
                 return self
             if self._closed:
                 raise RuntimeError("PageFetch has already been closed")
-            ensure_runtime_requirements()
+            await asyncio.to_thread(
+                ensure_runtime_requirements,
+                needs_browser=self.config.mode in ("auto", "browser"),
+            )
             if self._cache:
                 try:
                     await self._cache.start()
@@ -127,6 +134,19 @@ class PageFetch:
         async with self._lifecycle_lock:
             if self._closed:
                 return
+            # Wait for in-flight fetches to drain before tearing down resources
+            for _ in range(50):  # up to ~5 seconds
+                async with self._active_fetches_lock:
+                    if self._active_fetches == 0:
+                        break
+                await asyncio.sleep(0.1)
+            async with self._active_fetches_lock:
+                remaining = self._active_fetches
+            if remaining > 0:
+                logger.warning(
+                    "close timed out waiting for %d in-flight fetch(es); tearing down resources anyway",
+                    remaining,
+                )
             browser_results = await asyncio.gather(
                 *(browser.close() for browser in self._browser_fetchers.values()),
                 return_exceptions=True,
@@ -201,17 +221,40 @@ class PageFetch:
             return result
 
         await self.start()
-        ttl = self.config.cache_ttl if cache_ttl is None else parse_duration(cache_ttl)
+        if self._closed:
+            return self._finish_error(
+                url=normalized_url,
+                proxy=selected_proxy,
+                error=FetchErrorInfo("client_closed", "PageFetch client has been closed", False),
+                started_at=started_at,
+                should_raise=should_raise,
+            )
+        try:
+            ttl = self.config.cache_ttl if cache_ttl is None else parse_duration(cache_ttl)
+        except (TypeError, ValueError) as exc:
+            result = self._finish_error(
+                url=normalized_url,
+                proxy=selected_proxy,
+                error=FetchErrorInfo("invalid_cache_ttl", f"Invalid cache_ttl: {exc}", False, type(exc).__name__),
+                started_at=started_at,
+                should_raise=should_raise,
+            )
+            result.duration_ms = round((time.perf_counter() - started_at) * 1000, 2)
+            return result
         cache_key = build_cache_key(
             normalized_url,
             mode=selected_mode,
             proxy=selected_proxy,
             settings={
+                "block_images": self.config.block_images,
                 "browser_timeout": self.config.browser_timeout,
                 "confidence_threshold": self.config.confidence_threshold,
                 "headers": BROWSER_HEADERS,
+                "http_timeout": self.config.http_timeout,
                 "max_content_size": self.config.max_content_size,
+                "max_redirects": self.config.max_redirects,
                 "retries_browser": self.config.retries_browser,
+                "retries_http": self.config.retries_http,
             },
         )
         fetch_warnings = list(self._startup_warnings)
@@ -226,31 +269,37 @@ class PageFetch:
                 fetch_warnings.append("Cache read failed; content was fetched normally.")
                 logger.warning("cache read failed: %s", type(exc).__name__)
 
+        async with self._active_fetches_lock:
+            self._active_fetches += 1
         try:
-            if selected_mode == "browser":
-                result = await self._fetch_browser(normalized_url, selected_proxy, status_code=None)
-            else:
-                result = await self._fetch_http_or_auto(normalized_url, selected_mode, selected_proxy)
-        except (TransportFailure, ProxyConfigurationError) as exc:
-            error = exc.error if isinstance(exc, TransportFailure) else FetchErrorInfo(
-                "connection_error", str(exc), False, type(exc).__name__
-            )
-            result = self._finish_error(
-                url=normalized_url,
-                proxy=selected_proxy,
-                error=error,
-                status_code=getattr(exc, "status_code", None),
-                started_at=started_at,
-                should_raise=should_raise,
-            )
-        except Exception as exc:
-            result = self._finish_error(
-                url=normalized_url,
-                proxy=selected_proxy,
-                error=FetchErrorInfo("unknown_error", "An unexpected error occurred while fetching the page.", False, type(exc).__name__),
-                started_at=started_at,
-                should_raise=should_raise,
-            )
+            try:
+                if selected_mode == "browser":
+                    result = await self._fetch_browser(normalized_url, selected_proxy, status_code=None)
+                else:
+                    result = await self._fetch_http_or_auto(normalized_url, selected_mode, selected_proxy)
+            except (TransportFailure, ProxyConfigurationError) as exc:
+                error = exc.error if isinstance(exc, TransportFailure) else FetchErrorInfo(
+                    "connection_error", str(exc), False, type(exc).__name__
+                )
+                result = self._finish_error(
+                    url=normalized_url,
+                    proxy=selected_proxy,
+                    error=error,
+                    status_code=getattr(exc, "status_code", None),
+                    started_at=started_at,
+                    should_raise=should_raise,
+                )
+            except Exception as exc:
+                result = self._finish_error(
+                    url=normalized_url,
+                    proxy=selected_proxy,
+                    error=FetchErrorInfo("unknown_error", "An unexpected error occurred while fetching the page.", False, type(exc).__name__),
+                    started_at=started_at,
+                    should_raise=should_raise,
+                )
+        finally:
+            async with self._active_fetches_lock:
+                self._active_fetches -= 1
 
         result.duration_ms = round((time.perf_counter() - started_at) * 1000, 2)
         result.warnings[:0] = fetch_warnings
@@ -319,8 +368,14 @@ class PageFetch:
                     raise_on_error=_raise_on_error,
                 )
             except PageFetchError as exc:
+                # Use the raw item string — normalize_url would re-raise
+                # ValueError for the same invalid URL that caused the error.
+                try:
+                    result_url = normalize_url(item)
+                except ValueError:
+                    result_url = str(item)
                 return FetchResult(
-                    url=normalize_url(item),
+                    url=result_url,
                     success=False,
                     proxy_provider=proxy or self.config.proxy,
                     error=exc.error,
@@ -330,19 +385,32 @@ class PageFetch:
 
         fetched = await asyncio.gather(*(one(item) for item in unique))
         by_url = dict(zip(unique, fetched, strict=True))
-        return [by_url[item] for item in ordered]
+        # For duplicate URLs, return independent copies so callers
+        # do not accidentally alias mutable fields across entries.
+        from copy import deepcopy
+        results: list[FetchResult] = []
+        emitted: set[int] = set()
+        for item in ordered:
+            result = by_url[item]
+            if id(result) in emitted:
+                result = deepcopy(result)
+            emitted.add(id(result))
+            results.append(result)
+        return results
 
     async def _fetch_http_or_auto(self, url: str, mode: Literal["auto", "http", "browser"], proxy: str) -> FetchResult:
         try:
             response = await (await self._http_fetcher(proxy)).fetch(url)
-        except TransportFailure as exc:
-            if mode == "auto" and exc.error.retryable:
-                logger.info("HTTP transport failed; using browser for %s", url)
-                return await self._fetch_browser(url, proxy, status_code=None)
+        except TransportFailure:
+            # Timeouts, DNS failures, disconnects, and other transport errors are not
+            # fixed by rendering in a browser — surface them immediately instead of
+            # waiting on an expensive Camoufox navigation that will fail the same way.
             raise
         if response.status_code >= 400:
             retryable = response.status_code in RETRYABLE_STATUS_CODES
-            if mode == "auto" and (response.status_code in BLOCKED_STATUS_CODES or retryable):
+            # Only anti-bot / rate-limit responses benefit from a stealth browser.
+            # 4xx like 404 and 5xx server errors should fail fast at the HTTP layer.
+            if mode == "auto" and response.status_code in BLOCKED_STATUS_CODES:
                 logger.info("HTTP %s; using browser for %s", response.status_code, url)
                 return await self._fetch_browser(url, proxy, status_code=response.status_code)
             code = "blocked" if response.status_code in BLOCKED_STATUS_CODES else "http_error"
@@ -358,9 +426,18 @@ class PageFetch:
             return self._result_from_xml(url, response, proxy)
         if content_type.startswith("text/plain"):
             return self._result_from_text(url, response, proxy)
+        if not self._is_html_like(content_type) and not self._looks_like_html(response.content):
+            raise TransportFailure(
+                FetchErrorInfo(
+                    "unsupported_content_type",
+                    f"Content type {content_type!r} is not HTML; cannot process with HTTP/auto mode",
+                    False,
+                ),
+                status_code=response.status_code,
+            )
         html = self._decode(response)
         raw_soup = BeautifulSoup(html, "lxml")
-        report = analyze_html(html)
+        report = analyze_html(html, soup=raw_soup)
         if mode == "auto" and report.score < self.config.confidence_threshold:
             logger.info("HTTP confidence %.3f; using browser for %s", report.score, url)
             try:
@@ -377,6 +454,7 @@ class PageFetch:
                     method="http",
                     response_headers=response.headers,
                     soup=raw_soup,
+                    confidence=report,
                 )
                 available.warnings.extend(
                     [
@@ -397,6 +475,7 @@ class PageFetch:
                     method="http",
                     response_headers=response.headers,
                     soup=raw_soup,
+                    confidence=report,
                 )
                 available.warnings.extend(
                     [
@@ -418,6 +497,7 @@ class PageFetch:
             method="http",
             response_headers=response.headers,
             soup=raw_soup,
+            confidence=report,
         )
         if mode == "http" and report.score < self.config.confidence_threshold:
             result.warnings.append("HTTP content may be incomplete; browser fallback is disabled.")
@@ -436,9 +516,10 @@ class PageFetch:
             proxy=proxy,
             method="browser",
             soup=raw_soup,
+            confidence=response.confidence,
         )
         result.warnings.extend(response.warnings)
-        report = analyze_html(response.html)
+        report = response.confidence
         if response.status_code is not None and response.status_code >= 400:
             result.success = False
             code = "blocked" if response.status_code in BLOCKED_STATUS_CODES else "http_error"
@@ -493,6 +574,7 @@ class PageFetch:
                     proxy=resolve_proxy(provider),
                     max_content_size=self.config.max_content_size,
                     confidence_threshold=self.config.confidence_threshold,
+                    block_images=self.config.block_images,
                 )
         return self._browser_fetchers[provider]
 
@@ -509,9 +591,10 @@ class PageFetch:
         method: str,
         response_headers: httpx.Headers | None = None,
         soup: BeautifulSoup | None = None,
+        confidence: ConfidenceReport | None = None,
     ) -> FetchResult:
         try:
-            processed = process_html(html, final_url, response_headers, soup=soup)
+            processed = process_html(html, final_url, response_headers, soup=soup, confidence=confidence)
         except Exception as exc:
             raise TransportFailure(
                 FetchErrorInfo("parse_error", "HTML content could not be processed", False, type(exc).__name__)
@@ -616,7 +699,27 @@ class PageFetch:
 
     @staticmethod
     def _is_xml(content_type: str) -> bool:
-        return any(value in content_type for value in XML_TYPES)
+        if content_type in ("application/xml", "text/xml"):
+            return True
+        # Match XML-based subtypes like application/atom+xml, image/svg+xml
+        # but NOT application/xhtml+xml (which is HTML5, not generic XML).
+        return "+xml" in content_type and content_type != "application/xhtml+xml"
+
+    @staticmethod
+    def _is_html_like(content_type: str) -> bool:
+        """Return True when *content_type* should be processed as HTML."""
+        return content_type in ("text/html", "application/xhtml+xml")
+
+    @staticmethod
+    def _looks_like_html(content: bytes) -> bool:
+        """Heuristic HTML sniff for responses with ambiguous content types."""
+        stripped = content.lstrip()
+        if not stripped:
+            return False
+        return stripped.startswith(b"<") and any(
+            marker in stripped[:512].lower()
+            for marker in (b"<!doctype html", b"<html", b"<head", b"<body", b"<title", b"<meta", b"<div", b"<p", b"<a ")
+        )
 
     @staticmethod
     def _validate_fetch_options(mode: str, proxy: str) -> None:

{pagefetch-0.3.0 → pagefetch-0.5.0}/pagefetch/config.py

@@ -49,6 +49,7 @@ class PageFetchConfig:
     max_redirects: int = 10
     max_content_size: int = 25 * 1024 * 1024
     confidence_threshold: float = 0.80
+    block_images: bool = True
     raise_on_error: bool = False
 
     @classmethod
@@ -73,7 +74,12 @@ class PageFetchConfig:
         flat: dict[str, Any] = {}
         for section in ("proxy", "cache"):
             if section in resolved and isinstance(resolved[section], dict):
-                flat.update(resolved.pop(section))
+                section_data = resolved.pop(section)
+                # A nested proxy section may use "provider" to select the provider;
+                # map it to the top-level "proxy" key so it is not silently dropped.
+                if section == "proxy" and "provider" in section_data:
+                    flat["proxy"] = section_data.pop("provider")
+                flat.update(section_data)
         flat.update(resolved)
 
         return cls.build(
@@ -91,6 +97,7 @@ class PageFetchConfig:
             max_redirects=flat.get("max_redirects", 10),
             max_content_size=flat.get("max_content_size", 25 * 1024 * 1024),
             confidence_threshold=flat.get("confidence_threshold", 0.80),
+            block_images=flat.get("block_images", True),
             raise_on_error=flat.get("raise_on_error", False),
         )
 
@@ -112,6 +119,7 @@ class PageFetchConfig:
         max_redirects: int = 10,
         max_content_size: int = 25 * 1024 * 1024,
         confidence_threshold: float = 0.80,
+        block_images: bool = True,
         raise_on_error: bool = False,
     ) -> PageFetchConfig:
         if not isinstance(mode, str) or mode not in VALID_MODES:
@@ -146,6 +154,8 @@ class PageFetchConfig:
             raise ValueError("confidence_threshold must be between 0 and 1")
         if not isinstance(cache_enabled, bool) or not isinstance(raise_on_error, bool):
             raise ValueError("cache_enabled and raise_on_error must be booleans")
+        if not isinstance(block_images, bool):
+            raise ValueError("block_images must be a boolean")
         ttl = parse_duration(cache_ttl)
         path = Path(cache_path).expanduser() if cache_path is not None else user_cache_path("pagefetch") / "cache.sqlite3"
         return cls(
@@ -163,5 +173,6 @@ class PageFetchConfig:
             max_redirects=max_redirects,
             max_content_size=max_content_size,
             confidence_threshold=float(confidence_threshold),
+            block_images=block_images,
             raise_on_error=raise_on_error,
         )

{pagefetch-0.3.0 → pagefetch-0.5.0}/pagefetch/fetching/__init__.py

@@ -2,7 +2,7 @@
 
 from .browser import BrowserFetcher, BrowserResponse
 from .http import HTTPFetcher, HTTPResponse, TransportFailure
-from .readiness import controlled_scroll, wait_for_stability
+from .readiness import PageMetrics, controlled_scroll, in_page_metrics, wait_for_stability
 
 __all__ = [
     "BrowserFetcher",
@@ -10,6 +10,8 @@ __all__ = [
     "HTTPFetcher",
     "HTTPResponse",
     "TransportFailure",
+    "PageMetrics",
     "controlled_scroll",
+    "in_page_metrics",
     "wait_for_stability",
 ]