fetta 0.1.2__tar.gz → 0.3.0__tar.gz

{fetta-0.1.2 → fetta-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fetta
-Version: 0.1.2
+Version: 0.3.0
 Summary: Tool that fetches web pages for assistants
 Author: Sergei
 License-Expression: MIT
@@ -10,7 +10,7 @@ Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: beautifulsoup4>=4.15.0,<5
 Requires-Dist: playwright==1.63.0
-Requires-Dist: requests>=2.34.2,<3
+Requires-Dist: httpx>=0.28.1,<1
 Requires-Python: >=3.12
 Project-URL: Homepage, https://github.com/spapulin/fetta
 Project-URL: Repository, https://github.com/spapulin/fetta
@@ -41,7 +41,7 @@ from fetta import HttpFetcher
 async def fetch_content(url: str) -> str:
     async with HttpFetcher() as fetcher:
         page = await fetcher.fetch(url)
-        return page.text
+        return page.content.html
 
 # Sync entry point is top of a script, not inside an async function
 text = asyncio.run(fetch_content("https://example.com"))
@@ -53,8 +53,6 @@ sync programs only.
 
 ## Install
 
-`fetta` is not on PyPI yet. Install directly from GitHub.
-
 ### As a library
 
 With uv (recommended):
@@ -78,7 +76,7 @@ On macOS, drop the flag.
 
 Skip this step entirely if you only use `HttpFetcher`.
 
-Development version
+### Development version
 
 The latest unreleased code from `main`. May contain bugs, may
 break without notice. Use only if you need a fix that hasn't

{fetta-0.1.2 → fetta-0.3.0}/README.md

@@ -22,7 +22,7 @@ from fetta import HttpFetcher
 async def fetch_content(url: str) -> str:
     async with HttpFetcher() as fetcher:
         page = await fetcher.fetch(url)
-        return page.text
+        return page.content.html
 
 # Sync entry point is top of a script, not inside an async function
 text = asyncio.run(fetch_content("https://example.com"))
@@ -34,8 +34,6 @@ sync programs only.
 
 ## Install
 
-`fetta` is not on PyPI yet. Install directly from GitHub.
-
 ### As a library
 
 With uv (recommended):
@@ -59,7 +57,7 @@ On macOS, drop the flag.
 
 Skip this step entirely if you only use `HttpFetcher`.
 
-Development version
+### Development version
 
 The latest unreleased code from `main`. May contain bugs, may
 break without notice. Use only if you need a fix that hasn't

{fetta-0.1.2 → fetta-0.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fetta"
-version = "0.1.2"
+version = "0.3.0"
 description = "Tool that fetches web pages for assistants"
 readme = "README.md"
 license = "MIT"
@@ -14,7 +14,7 @@ requires-python = ">=3.12"
 dependencies = [
     "beautifulsoup4>=4.15.0,<5",
     "playwright==1.63.0",
-    "requests>=2.34.2,<3",
+    "httpx>=0.28.1,<1",
 ]
 
 [[project.authors]]
@@ -39,7 +39,7 @@ dev = [
     "jupyterlab>=4.6.4",
     "pytest>=9.1.1",
     "pytest-asyncio>=1.4.0",
-    "requests-mock>=1.12.1",
+    "respx>=0.23.1,<1",
     "ruff>=0.16.8",
 ]
 
@@ -50,6 +50,8 @@ module-root = "src"
 asyncio_mode = "auto"
 asyncio_default_fixture_loop_scope = "session"
 asyncio_default_test_loop_scope = "session"
+markers = ["integration: tests that hit the real network"]
+addopts = "-m 'not integration'"
 testpaths = ["tests"]
 
 [tool.ruff]

{fetta-0.1.2 → fetta-0.3.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "fetta"
-version = "0.1.2"
+version = "0.3.0"
 description = "Tool that fetches web pages for assistants"
 readme = "README.md"
 authors = [{ name = "Sergei" }]
@@ -15,7 +15,7 @@ requires-python = ">=3.12"
 dependencies = [
     "beautifulsoup4>=4.15.0,<5",
     "playwright==1.63.0",
-    "requests>=2.34.2,<3"
+    "httpx>=0.28.1,<1"
 ]
 
 [project.scripts]
@@ -37,7 +37,7 @@ dev = [
     "jupyterlab>=4.6.4",
     "pytest>=9.1.1",
     "pytest-asyncio>=1.4.0",
-    "requests-mock>=1.12.1",
+    "respx>=0.23.1,<1",
     "ruff>=0.16.8",
 ]
 
@@ -48,6 +48,8 @@ module-root = "src"
 asyncio_mode = "auto"
 asyncio_default_fixture_loop_scope = "session"
 asyncio_default_test_loop_scope = "session"
+markers = ["integration: tests that hit the real network"]
+addopts = "-m 'not integration'"
 testpaths = ["tests"]
 
 [tool.ruff]

fetta-0.3.0/src/fetta/__init__.py

@@ -0,0 +1,19 @@
+from .browser import BrowserManager
+from .fetchers import BrowserFetcher, HttpFetcher, SmartFetcher
+from .json_fetcher import JsonFetcher
+from .models import Link, Image, PageContent, PageMetadata, FetchResult
+from .protocol import FetcherProtocol
+
+__all__ = [
+    "BrowserManager",
+    "PageContent",
+    "Link",
+    "Image",
+    "PageMetadata",
+    "FetchResult",
+    "BrowserFetcher",
+    "HttpFetcher",
+    "FetcherProtocol",
+    "JsonFetcher",
+    "SmartFetcher"
+]

fetta-0.3.0/src/fetta/base.py

@@ -0,0 +1,78 @@
+from abc import ABC, abstractmethod
+from typing import Any
+
+from .models import FetchResult, PageContent, PageMetadata, FetcherKind
+from .page_parser.constants import DEFAULT_SECTIONS
+from .page_parser.parser import parse_html
+
+
+class BaseFetcher(ABC):
+    """
+    Abstract base class for fetchers.
+    """
+
+    fetcher_kind: FetcherKind
+
+    @abstractmethod
+    async def fetch(self, url: str, **kwargs: Any) -> FetchResult:
+        """Fetch a page and return its parsed content."""
+        pass
+
+    @abstractmethod
+    async def close(self) -> None:
+        """Release resources (HTTP connections, browser context, etc.)."""
+        pass
+
+    async def __aenter__(self) -> "BaseFetcher":
+        return self
+
+    async def __aexit__(self, exc_type, exc, tb) -> None:
+        await self.close()
+
+    def _parse(
+        self,
+        html: str,
+        url: str,
+        status: int,
+        title: str = "",
+        csr_data: list[dict] = None,
+        include: tuple[str, ...] = DEFAULT_SECTIONS
+    ) -> FetchResult:
+        """
+        Turn raw HTML into a PageContent.
+
+        Extracts the title (if not already provided), strips boilerplate
+        tags, and normalizes the remaining text.
+
+        Args:
+            html: Raw HTML as returned by the transport.
+            url: Original URL (stored on the result for traceability).
+            status: HTTP status code.
+            title: Pre-extracted title. If empty, the <title> tag is used.
+        """
+        parsed_content = parse_html(
+            html=html,
+            page_url=url,
+            include=include
+        )
+        page_metadata = parsed_content.get("metadata") or {}
+
+        return FetchResult(
+            url=url,
+            content=PageContent(
+                html=parsed_content.get("html") or "",
+                text=parsed_content.get("text"),
+                raw_html=parsed_content.get("raw_html"),
+                raw_text=parsed_content.get("raw_text"),
+                metadata=parsed_content.get("metadata"),
+                links=tuple(parsed_content.get("links") or []),
+                images=parsed_content.get("images") or [],
+                csr_data=tuple(csr_data) if csr_data is not None else None
+            ),
+            title=page_metadata.get("title"),
+            icon=page_metadata.get("icon"),
+            image=page_metadata.get("image"),
+            description=page_metadata.get("description"),
+            status=status,
+            fetcher_kind=self.fetcher_kind
+        )

fetta-0.3.0/src/fetta/browser.py

@@ -0,0 +1,54 @@
+from asyncio import Semaphore
+from contextlib import asynccontextmanager
+from typing import AsyncIterator
+
+from playwright.async_api import (
+    Browser,
+    Playwright,
+    async_playwright,
+    ViewportSize,
+    Page
+)
+
+
+class BrowserManager:
+    def __init__(self, max_contexts: int = 10) -> None:
+        self._semaphore = Semaphore(max_contexts)
+        self._playwright: Playwright | None = None
+        self._browser: Browser | None = None
+
+    async def start(self) -> None:
+        self._playwright = await async_playwright().start()
+        self._browser = await self._playwright.chromium.launch(headless=True)
+
+    async def stop(self) -> None:
+        if self._browser is not None:
+            await self._browser.close()
+            self._browser = None
+        if self._playwright is not None:
+            await self._playwright.stop()
+            self._playwright = None
+
+    @asynccontextmanager
+    async def page(self) -> AsyncIterator[Page]:
+        if self._browser is None:
+            raise RuntimeError("BrowserManager is not started")
+        async with self._semaphore:
+            context = await self._browser.new_context(
+                user_agent=(
+                    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                    "AppleWebKit/537.36 (KHTML, like Gecko) "
+                    "Chrome/153.0.0.0 Safari/537.36"
+                ),
+                viewport=ViewportSize(**{"width": 1920, "height": 1080}),
+                locale="en-GB",
+                extra_http_headers={
+                    "Accept-Language": "en-GB,en;q=0.9,en-US;q=0.8,en;q=0.7",
+                },
+            )
+            page = await context.new_page()
+            try:
+                yield page
+            finally:
+                await context.close()
+

fetta-0.3.0/src/fetta/fetchers.py

@@ -0,0 +1,91 @@
+from typing import Any
+
+from .httpx_fetcher import HttpxFetcher
+from .protocol import FetcherProtocol
+from .base import BaseFetcher
+from .browser import BrowserManager
+from .models import FetchResult, FetcherKind
+from .playwright_fetcher import PlaywrightFetcher
+from .smart_fetcher import VanillaSmartFetcher
+
+
+class HttpFetcher(BaseFetcher):
+    """
+    Fetches pages over plain HTTP (no browser, no JavaScript).
+
+    A facade over pluggable HTTP engines. By default, it uses
+    HttpxFetcher (sync `requests` offloaded to a worker thread).
+    """
+
+    fetcher_kind: FetcherKind = "http"
+
+    def __init__(
+        self,
+        *,
+        engine: FetcherProtocol | None = None,
+    ) -> None:
+        self._engine: FetcherProtocol = engine or HttpxFetcher()
+
+    async def fetch(self, url: str, **kwargs: Any) -> FetchResult:
+        return await self._engine.fetch(url, **kwargs)
+
+    async def close(self) -> None:
+        await self._engine.close()
+
+
+class BrowserFetcher(BaseFetcher):
+    """
+    Fetches pages through a real browser.
+
+    The engine is an implementation detail. Currently, Playwright is
+    used; alternative engines can be injected via the `engine` argument.
+    """
+
+    fetcher_kind: FetcherKind = "browser"
+
+    def __init__(
+        self,
+        manager: BrowserManager,
+        *,
+        engine: FetcherProtocol | None = None
+    ) -> None:
+        self._engine: FetcherProtocol = engine or PlaywrightFetcher(manager)
+
+    async def fetch(self, url: str, **kwargs: Any) -> FetchResult:
+        return await self._engine.fetch(url, **kwargs)
+
+    async def close(self) -> None:
+        await self._engine.close()
+
+
+
+class SmartFetcher(BaseFetcher):
+    """
+    Fetches pages with the cheapest fetcher that can render them.
+
+    The page is first fetched via the HTML fetcher. If the response
+    looks client-side rendered, the browser fetcher is used instead.
+    """
+
+    def __init__(
+        self,
+        manager: BrowserManager | None = None,
+        *,
+        html_fetcher: FetcherProtocol | None = None,
+        browser_fetcher: FetcherProtocol | None = None,
+    ) -> None:
+        html_engine: FetcherProtocol = html_fetcher or HttpxFetcher()
+        if not browser_fetcher and not manager:
+            browser_engine = None
+        else:
+            browser_engine = browser_fetcher or PlaywrightFetcher(manager)
+        self._engine = VanillaSmartFetcher(
+            html_fetcher=html_engine,
+            browser_fetcher=browser_engine
+        )
+
+    async def fetch(self, url: str, **kwargs: Any) -> FetchResult:
+        return await self._engine.fetch(url, **kwargs)
+
+    async def close(self) -> None:
+        await self._engine.close()

fetta-0.3.0/src/fetta/httpx_fetcher.py

@@ -0,0 +1,69 @@
+from typing import Any
+
+import httpx
+
+from .models import FetchResult, FetcherKind
+from .base import BaseFetcher
+from .page_parser.constants import DEFAULT_SECTIONS
+
+
+class HttpxFetcher(BaseFetcher):
+    """
+    Native async HTTP fetcher using httpx.AsyncClient.
+
+    Connection pooling is handled by the single client instance; do not
+    create a new client per request (that defeats pooling and adds
+    TLS-handshake overhead).
+    """
+
+    fetcher_kind: FetcherKind = "http"
+
+    def __init__(self, *, timeout: float = 10.0, http2: bool = False) -> None:
+        self._client = httpx.AsyncClient(
+            timeout=timeout,
+            follow_redirects=True,
+            http2=http2,
+            headers={
+                "User-Agent": (
+                    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                    "AppleWebKit/537.36 (KHTML, like Gecko) "
+                    "Chrome/149.0.0.0 Safari/537.36"
+                ),
+                "Accept": (
+                    "text/html,application/xhtml+xml,application/xml;q=0.9,"
+                    "image/avif,image/webp,*/*;q=0.8"
+                ),
+                "Accept-Language": "en-GB,en;q=0.9,en-US;q=0.8,en;q=0.7",
+                # Let httpx negotiate Accept-Encoding automatically
+                "Connection": "keep-alive",
+                "Upgrade-Insecure-Requests": "1",
+                "Sec-Fetch-Dest": "document",
+                "Sec-Fetch-Mode": "navigate",
+                "Sec-Fetch-Site": "none",
+                "Sec-Fetch-User": "?1",
+                "Cache-Control": "max-age=0",
+                "sec-ch-ua": '"Chromium";v="149", "Not)A;Brand";v="24"',
+                "sec-ch-ua-mobile": "?0",
+                "sec-ch-ua-platform": '"Windows"',
+            },
+        )
+
+    async def fetch(
+        self,
+        url: str,
+        *,
+        include: tuple[str, ...] = DEFAULT_SECTIONS,
+        **kwargs: Any
+    ) -> FetchResult:
+        response = await self._client.get(url)
+        response.raise_for_status()
+
+        return self._parse(
+            html=response.text,
+            url=url,
+            status=response.status_code,
+            include=include,
+        )
+
+    async def close(self) -> None:
+        await self._client.aclose()

{fetta-0.1.2 → fetta-0.3.0}/src/fetta/json_fetcher.py

@@ -1,12 +1,14 @@
-from fetta.base import BaseFetcher
-from fetta.browser import BrowserManager
-from fetta.models import PageContent
-
+from .base import BaseFetcher
+from .browser import BrowserManager
+from .models import FetchResult, FetcherKind
 
 _BLOCK = {"image", "media", "font"}
 
 
 class JsonFetcher(BaseFetcher):
+
+    fetcher_kind: FetcherKind = "browser"
+
     def __init__(self, browser: BrowserManager) -> None:
         self._browser = browser
 
@@ -16,8 +18,9 @@ class JsonFetcher(BaseFetcher):
         *,
         wait_until: str = "load",
         timeout_ms: int = 30_000,
+        include: tuple[str, ...] = ("metadata",),
         **kwargs,
-    ) -> PageContent:
+    ) -> FetchResult:
 
         json_responses: list[dict] = []
 
@@ -50,7 +53,14 @@ class JsonFetcher(BaseFetcher):
             html = await page.content()
             title = await page.title()
 
-        return self._parse(html, url, response.status if response else 0, title, json_responses)
+        return self._parse(
+            html=html,
+            url=url,
+            status=response.status if response else 0,
+            title=title,
+            csr_data=json_responses,
+            include=include
+        )
 
     async def close(self) -> None:
         # Browser lifecycle is owned by BrowserManager, not by the fetcher.

fetta-0.3.0/src/fetta/models.py

@@ -0,0 +1,57 @@
+from dataclasses import dataclass
+from typing import TypedDict, Literal
+
+
+class PageMetadata(TypedDict):
+    page_url: str
+    canonical_url: str | None
+    icon: str | None
+    title: str | None
+    description: str | None
+    site_name: str | None
+    og_image: str | None
+    og_type: str | None
+    ld_json: tuple | None
+
+
+@dataclass(frozen=True, slots=True)
+class Link:
+    text: str
+    url: str
+    external: bool
+    section: str | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class Image:
+    src: str
+    alt: str | None = None
+    title: str | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class PageContent:
+    html: str
+    text: str | None
+    raw_html: str | None
+    raw_text: str | None
+    metadata: PageMetadata | None
+    links: tuple[Link, ...]
+    images: tuple[Image, ...]
+    csr_data: tuple[dict, ...] | None
+
+
+FetcherKind = Literal["http", "browser"]
+
+@dataclass(frozen=True, slots=True)
+class FetchResult:
+    url: str
+    status: int
+    fetcher_kind: FetcherKind
+    # Meta
+    title: str | None
+    icon: str | None
+    description: str | None
+    image: str | None
+    # Content
+    content: PageContent

fetta-0.3.0/src/fetta/page_parser/cleaning.py

@@ -0,0 +1,76 @@
+import copy
+import re
+
+from bs4 import Comment
+from bs4.element import Tag
+
+from .constants import DROP_TAGS, NOISE_RE, KEEP_TAGS, KEEP_ATTRS
+
+
+STRUCTURAL = frozenset({"html", "head", "body", "main", "article"})
+
+
+def clean_html(node: Tag, *, inplace: bool = False) -> Tag:
+    """Drop scripts, styles, comments, and noise blocks.
+
+    Returns a new soup unless inplace=True
+    """
+
+    if not inplace:
+        node = copy.deepcopy(node)
+
+    for tag in node(DROP_TAGS):
+        if not tag.decomposed:
+            tag.decompose()
+
+    for comment in node.find_all(string=lambda s: isinstance(s, Comment)):
+        comment.extract()
+
+    for tag in node.find_all(True):
+        if tag.name in STRUCTURAL or tag.decomposed or not tag.attrs:
+            continue
+
+        attrs = " ".join(
+            filter(
+                None,
+                (
+                    " ".join(tag.get("class", [])),
+                    tag.get("id", ""),
+                    tag.get("role", ""),
+                ),
+            )
+        )
+        if attrs and NOISE_RE.search(attrs):
+            tag.decompose()
+
+    return node
+
+
+def minify_html(
+    node: Tag, *, max_chars: int = 25_000, inplace: bool = False
+) -> tuple[str, str | None]:
+    """Strip unwanted tags/attrs and collapse whitespace.
+
+    Returns a string. Input tree is copied unless inplace=True
+    """
+    if not inplace:
+        node = copy.deepcopy(node)
+
+    for tag in list(node.find_all(True)):
+        if tag.name not in KEEP_TAGS:
+            tag.unwrap()
+            continue
+
+        for attr in list(tag.attrs):
+            if attr not in KEEP_ATTRS or tag.attrs[attr] in ("", None):
+                del tag[attr]
+
+    # Flatten bare wrappers
+    for tag in node.find_all(["div", "span"]):
+        if not tag.attrs:
+            tag.unwrap()
+
+    text = str(node)
+    text = re.sub(r"\s+", " ", text)
+    text = re.sub(r">\s+<", "><", text)
+    return text.strip()[:max_chars], node.get_text(" ", strip=True)[:max_chars]