pagefetch 0.1.0__py3-none-any.whl

pagefetch/__init__.py

@@ -0,0 +1,29 @@
+"""Public PageFetch API."""
+
+import logging
+
+from .bootstrap import RuntimeBootstrapError, ensure_runtime_requirements
+
+# NOTE: ensure_runtime_requirements() is NOT called here to avoid side effects
+# on import (network calls / subprocess execution).  It is called lazily inside
+# PageFetch.start() before the first fetch operation.  If you use the library
+# without calling PageFetch.start() directly, call
+#   pagefetch.ensure_runtime_requirements()
+# explicitly, or set PAGEFETCH_AUTO_INSTALL=0 to disable auto-installation.
+logging.getLogger("pagefetch").addHandler(logging.NullHandler())
+
+from .client import PageFetch  # noqa: E402
+from .exceptions import PageFetchError  # noqa: E402
+from .models import FetchErrorInfo, FetchResult, ImageInfo, LinkInfo  # noqa: E402
+
+__all__ = [
+    "FetchErrorInfo",
+    "FetchResult",
+    "ImageInfo",
+    "LinkInfo",
+    "PageFetch",
+    "PageFetchError",
+    "RuntimeBootstrapError",
+]
+
+__version__ = "0.1.0"

pagefetch/__main__.py

@@ -0,0 +1,4 @@
+from .cli import main
+
+raise SystemExit(main())
+

pagefetch/bootstrap.py

@@ -0,0 +1,113 @@
+"""Runtime dependency bootstrap for direct source and CLI execution."""
+
+from __future__ import annotations
+
+import importlib
+import importlib.util
+import os
+import subprocess
+import sys
+import threading
+from collections.abc import Callable
+
+_RUNTIME_REQUIREMENTS = (
+    ("aiosqlite", "aiosqlite>=0.20"),
+    ("bs4", "beautifulsoup4>=4.12"),
+    ("camoufox", "camoufox>=0.4"),
+    ("httpx", "httpx[http2,socks]>=0.27"),
+    ("h2", "httpx[http2,socks]>=0.27"),
+    ("socksio", "httpx[http2,socks]>=0.27"),
+    ("lxml", "lxml>=5.2"),
+    ("platformdirs", "platformdirs>=4.2"),
+    ("pypdf", "pypdf>=5.0"),
+    ("yaml", "pyyaml>=6.0"),
+    ("tldextract", "tldextract>=5.1"),
+)
+_FALSE_VALUES = frozenset({"0", "false", "no", "off"})
+_bootstrap_lock = threading.Lock()
+_bootstrap_complete = False
+
+
+class RuntimeBootstrapError(RuntimeError):
+    """Raised when PageFetch cannot install a required runtime component."""
+
+
+def auto_install_enabled() -> bool:
+    """Return whether automatic startup installation is enabled."""
+    value = os.getenv("PAGEFETCH_AUTO_INSTALL", "1").strip().lower()
+    return value not in _FALSE_VALUES
+
+
+def _missing_requirements(
+    find_spec: Callable[[str], object | None] = importlib.util.find_spec,
+) -> list[str]:
+    missing: list[str] = []
+    for module, requirement in _RUNTIME_REQUIREMENTS:
+        if find_spec(module) is None and requirement not in missing:
+            missing.append(requirement)
+    return missing
+
+
+def install_python_requirements() -> None:
+    """Install Python runtime packages that are absent from this interpreter."""
+    missing = _missing_requirements()
+    if not missing:
+        return
+    print(
+        f"[pagefetch] Installing missing Python requirements: {', '.join(missing)}",
+        file=sys.stderr,
+    )
+    command = [
+        sys.executable,
+        "-m",
+        "pip",
+        "install",
+        "--disable-pip-version-check",
+        *missing,
+    ]
+    try:
+        subprocess.run(command, check=True)
+    except (OSError, subprocess.CalledProcessError) as exc:
+        raise RuntimeBootstrapError(
+            "PageFetch could not install its Python requirements. "
+            "Run 'python -m pip install .' manually or set PAGEFETCH_AUTO_INSTALL=0 "
+            "to disable automatic installation."
+        ) from exc
+    importlib.invalidate_caches()
+    unresolved = _missing_requirements()
+    if unresolved:
+        raise RuntimeBootstrapError(
+            f"Requirements remain unavailable after installation: {', '.join(unresolved)}"
+        )
+
+
+def install_camoufox_browser() -> None:
+    """Download or update the Camoufox browser binary when it is unavailable."""
+    try:
+        from camoufox.pkgman import camoufox_path
+
+        try:
+            camoufox_path(download_if_missing=False)
+            return
+        except Exception:
+            print("[pagefetch] Installing the Camoufox browser runtime...", file=sys.stderr)
+        camoufox_path(download_if_missing=True)
+    except Exception as exc:
+        raise RuntimeBootstrapError(
+            "PageFetch could not install the Camoufox browser runtime. "
+            "Run 'python -m camoufox fetch' manually or set PAGEFETCH_AUTO_INSTALL=0 "
+            "to disable automatic installation."
+        ) from exc
+
+
+def ensure_runtime_requirements() -> None:
+    """Install all runtime requirements once at process startup."""
+    global _bootstrap_complete
+    if not auto_install_enabled():
+        return
+    with _bootstrap_lock:
+        if _bootstrap_complete:
+            return
+        install_python_requirements()
+        install_camoufox_browser()
+        _bootstrap_complete = True

pagefetch/cache/__init__.py

@@ -0,0 +1,7 @@
+"""Persistent result cache."""
+
+from .keys import build_cache_key
+from .sqlite import SQLiteCache
+
+__all__ = ["SQLiteCache", "build_cache_key"]
+

pagefetch/cache/keys.py

@@ -0,0 +1,28 @@
+"""Stable, credential-safe cache keys."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from typing import Any
+
+from ..utils.urls import normalize_url
+
+
+def build_cache_key(
+    url: str,
+    *,
+    mode: str,
+    proxy: str,
+    settings: dict[str, Any] | None = None,
+) -> str:
+    payload = {
+        "url": normalize_url(url),
+        "mode": mode,
+        "proxy": proxy,
+        "processing_version": 1,
+        "settings": settings or {},
+    }
+    raw = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
+    return hashlib.sha256(raw.encode("utf-8")).hexdigest()
+

pagefetch/cache/sqlite.py

@@ -0,0 +1,103 @@
+"""Async SQLite cache implementation."""
+
+from __future__ import annotations
+
+import json
+import time
+from pathlib import Path
+
+import aiosqlite
+
+from ..models import FetchResult
+
+
+class SQLiteCache:
+    """Persistent cache for successful fetch results."""
+
+    def __init__(self, path: Path) -> None:
+        self.path = path
+        self._db: aiosqlite.Connection | None = None
+
+    async def start(self) -> None:
+        if self._db is not None:
+            return
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self._db = await aiosqlite.connect(self.path)
+        await self._db.execute("PRAGMA journal_mode=WAL")
+        await self._db.execute(
+            """
+            CREATE TABLE IF NOT EXISTS fetch_cache (
+                cache_key TEXT PRIMARY KEY,
+                payload TEXT NOT NULL,
+                created_at REAL NOT NULL,
+                expires_at REAL NOT NULL
+            )
+            """
+        )
+        await self._db.commit()
+
+    async def get(self, key: str) -> FetchResult | None:
+        """Return the cached result for *key*, or ``None`` when the key is missing
+        or the cached entry has expired.
+
+        Expired entries are automatically deleted from the database before
+        returning ``None``.
+        """
+        if self._db is None:
+            raise RuntimeError("cache has not been started")
+        cursor = await self._db.execute(
+            "SELECT payload, expires_at FROM fetch_cache WHERE cache_key = ?", (key,)
+        )
+        row = await cursor.fetchone()
+        await cursor.close()
+        if not row:
+            return None
+        if row[1] <= time.time():
+            await self._db.execute("DELETE FROM fetch_cache WHERE cache_key = ?", (key,))
+            await self._db.commit()
+            return None
+        try:
+            result = FetchResult.from_dict(json.loads(row[0]))
+        except (TypeError, ValueError, json.JSONDecodeError):
+            await self._db.execute("DELETE FROM fetch_cache WHERE cache_key = ?", (key,))
+            await self._db.commit()
+            return None
+        result.from_cache = True
+        result.fetch_method = "cache"
+        return result
+
+    async def set(self, key: str, result: FetchResult, ttl: int) -> None:
+        """Persist *result* in the cache with the given *ttl* (in seconds).
+
+        Only successful results are cached.  If the result cannot be serialized
+        a :exc:`RuntimeError` is raised with details about the failure.
+        """
+        if self._db is None:
+            raise RuntimeError("cache has not been started")
+        if not result.success:
+            return
+        now = time.time()
+        try:
+            payload = result.json(include_html=True)
+        except (TypeError, ValueError) as exc:
+            raise RuntimeError(
+                f"Failed to serialize fetch result for cache key {key!r}: {exc}"
+            ) from exc
+        await self._db.execute(
+            """
+            INSERT INTO fetch_cache(cache_key, payload, created_at, expires_at)
+            VALUES (?, ?, ?, ?)
+            ON CONFLICT(cache_key) DO UPDATE SET
+                payload=excluded.payload,
+                created_at=excluded.created_at,
+                expires_at=excluded.expires_at
+            """,
+            (key, payload, now, now + ttl),
+        )
+        await self._db.commit()
+
+    async def close(self) -> None:
+        if self._db is not None:
+            await self._db.close()
+            self._db = None
+

pagefetch/cli.py

@@ -0,0 +1,165 @@
+"""Small command-line interface for testing PageFetch."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import logging
+import sys
+from pathlib import Path
+
+from .client import PageFetch
+from .config import PageFetchConfig
+from .models import FetchResult
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="pagefetch", description="Fetch complete web page content")
+    parser.add_argument("input", metavar="URL_OR_FILE")
+    parser.add_argument("--format", choices=("markdown", "json", "html"), default="markdown")
+    parser.add_argument("-c", "--config", type=Path, metavar="PATH", help="Path to config.yaml")
+    parser.add_argument("--mode", choices=("auto", "http", "browser"), default=argparse.SUPPRESS)
+    parser.add_argument("--proxy", choices=("none", "decodo", "dataimpulse"), default=argparse.SUPPRESS)
+    parser.add_argument("-o", "--output", type=Path)
+    parser.add_argument("--include-html", action="store_true")
+    parser.add_argument(
+        "--cache-ttl",
+        metavar="DURATION",
+        default=argparse.SUPPRESS,
+        help="Cache time-to-live, e.g. 30s, 15m, 24h, 7d (default: 24h)",
+    )
+    parser.add_argument("--no-cache", action="store_true")
+    parser.add_argument("--http-concurrency", type=int, default=argparse.SUPPRESS)
+    parser.add_argument("--browser-concurrency", type=int, default=argparse.SUPPRESS)
+    parser.add_argument("--timeout", type=float, default=argparse.SUPPRESS)
+    parser.add_argument("--browser-timeout", type=float, default=argparse.SUPPRESS)
+    parser.add_argument("--debug", action="store_true")
+    return parser
+
+
+def _inputs(value: str) -> list[str]:
+    if value.startswith(("http://", "https://")):
+        return [value]
+    path = Path(value)
+    if not path.is_file():
+        # If it looks like a file path (has an extension or a directory separator)
+        # but doesn't exist, give a clear error instead of treating it as a URL.
+        if path.suffix or "/" in value or "\\" in value:
+            raise ValueError(f"file not found: {value!r}")
+        # Otherwise assume it's a URL (downstream validation will catch bad ones).
+        return [value]
+    return [line.strip() for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
+
+
+def _render(results: list[FetchResult], output_format: str, include_html: bool) -> str:
+    if output_format == "json":
+        values = [result.to_dict(include_html=include_html) for result in results]
+        return json.dumps(values, ensure_ascii=False, indent=2)
+    if output_format == "html":
+        documents = [result.html or result.text or "" for result in results]
+    else:
+        documents = [result.markdown or result.json() for result in results]
+    return "\n\n---\n\n".join(documents)
+
+
+def _build_config(args: argparse.Namespace) -> PageFetchConfig:
+    """Build configuration from YAML file (if provided) with CLI overrides."""
+    # Start from YAML if --config is given, otherwise use built-in defaults
+    if args.config:
+        config = PageFetchConfig.from_yaml(args.config)
+    else:
+        config = PageFetchConfig()
+
+    # CLI overrides — only apply when the user explicitly set the argument
+    overrides: dict[str, object] = {}
+    if hasattr(args, "mode"):
+        overrides["mode"] = args.mode
+    if hasattr(args, "proxy"):
+        overrides["proxy"] = args.proxy
+    if hasattr(args, "cache_ttl"):
+        overrides["cache_ttl"] = args.cache_ttl
+    if hasattr(args, "http_concurrency"):
+        overrides["http_concurrency"] = args.http_concurrency
+    if hasattr(args, "browser_concurrency"):
+        overrides["browser_concurrency"] = args.browser_concurrency
+    if hasattr(args, "timeout"):
+        overrides["http_timeout"] = args.timeout
+    if hasattr(args, "browser_timeout"):
+        overrides["browser_timeout"] = args.browser_timeout
+
+    if not overrides and not args.no_cache:
+        return config
+
+    # Rebuild with overrides applied
+    return PageFetchConfig.build(
+        mode=str(overrides.get("mode", config.mode)),
+        proxy=str(overrides.get("proxy", config.proxy)),
+        http_concurrency=int(overrides.get("http_concurrency", config.http_concurrency)),
+        browser_concurrency=int(overrides.get("browser_concurrency", config.browser_concurrency)),
+        cache_enabled=not args.no_cache if args.no_cache else config.cache_enabled,
+        cache_ttl=overrides.get("cache_ttl", config.cache_ttl),
+        cache_path=config.cache_path,
+        http_timeout=float(overrides.get("http_timeout", config.http_timeout)),
+        browser_timeout=float(overrides.get("browser_timeout", config.browser_timeout)),
+        retries_http=config.retries_http,
+        retries_browser=config.retries_browser,
+        max_redirects=config.max_redirects,
+        max_content_size=config.max_content_size,
+        confidence_threshold=config.confidence_threshold,
+        raise_on_error=config.raise_on_error,
+    )
+
+
+async def _run(args: argparse.Namespace) -> int:
+    urls = _inputs(args.input)
+    if not urls:
+        raise ValueError("the input file does not contain any URLs")
+    config = _build_config(args)
+    async with PageFetch(
+        mode=config.mode,
+        proxy=config.proxy,
+        cache_enabled=config.cache_enabled,
+        cache_ttl=config.cache_ttl,
+        http_concurrency=config.http_concurrency,
+        browser_concurrency=config.browser_concurrency,
+        http_timeout=config.http_timeout,
+        browser_timeout=config.browser_timeout,
+    ) as client:
+        results = await client.fetch_many(urls)
+    rendered = _render(results, args.format, args.include_html)
+    if args.output:
+        args.output.write_text(rendered, encoding="utf-8")
+    else:
+        try:
+            print(rendered)
+        except UnicodeEncodeError:
+            sys.stdout.buffer.write((rendered + "\n").encode("utf-8"))
+    total = len(results)
+    success_count = sum(1 for r in results if r.success)
+    cache_count = sum(1 for r in results if r.from_cache)
+    print(f"pagefetch: {total} URL(s) — {success_count} succeeded ({cache_count} from cache), {total - success_count} failed", file=sys.stderr)
+    if not results:
+        return 0
+    if success_count == 0:
+        return 1
+    if success_count < total:
+        return 3  # partial failure
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    if args.debug:
+        handler = logging.StreamHandler()
+        logging.getLogger("pagefetch").addHandler(handler)
+        logging.getLogger("pagefetch").setLevel(logging.DEBUG)
+    try:
+        return asyncio.run(_run(args))
+    except (ValueError, OSError) as exc:
+        print(f"pagefetch: {exc}", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())