fetta 0.1.2__tar.gz → 0.2.0__tar.gz

{fetta-0.1.2 → fetta-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fetta
-Version: 0.1.2
+Version: 0.2.0
 Summary: Tool that fetches web pages for assistants
 Author: Sergei
 License-Expression: MIT
@@ -10,7 +10,7 @@ Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: beautifulsoup4>=4.15.0,<5
 Requires-Dist: playwright==1.63.0
-Requires-Dist: requests>=2.34.2,<3
+Requires-Dist: httpx>=0.28.1,<1
 Requires-Python: >=3.12
 Project-URL: Homepage, https://github.com/spapulin/fetta
 Project-URL: Repository, https://github.com/spapulin/fetta
@@ -41,7 +41,7 @@ from fetta import HttpFetcher
 async def fetch_content(url: str) -> str:
     async with HttpFetcher() as fetcher:
         page = await fetcher.fetch(url)
-        return page.text
+        return page.content.html
 
 # Sync entry point is top of a script, not inside an async function
 text = asyncio.run(fetch_content("https://example.com"))
@@ -53,8 +53,6 @@ sync programs only.
 
 ## Install
 
-`fetta` is not on PyPI yet. Install directly from GitHub.
-
 ### As a library
 
 With uv (recommended):
@@ -78,7 +76,7 @@ On macOS, drop the flag.
 
 Skip this step entirely if you only use `HttpFetcher`.
 
-Development version
+### Development version
 
 The latest unreleased code from `main`. May contain bugs, may
 break without notice. Use only if you need a fix that hasn't

{fetta-0.1.2 → fetta-0.2.0}/README.md

@@ -22,7 +22,7 @@ from fetta import HttpFetcher
 async def fetch_content(url: str) -> str:
     async with HttpFetcher() as fetcher:
         page = await fetcher.fetch(url)
-        return page.text
+        return page.content.html
 
 # Sync entry point is top of a script, not inside an async function
 text = asyncio.run(fetch_content("https://example.com"))
@@ -34,8 +34,6 @@ sync programs only.
 
 ## Install
 
-`fetta` is not on PyPI yet. Install directly from GitHub.
-
 ### As a library
 
 With uv (recommended):
@@ -59,7 +57,7 @@ On macOS, drop the flag.
 
 Skip this step entirely if you only use `HttpFetcher`.
 
-Development version
+### Development version
 
 The latest unreleased code from `main`. May contain bugs, may
 break without notice. Use only if you need a fix that hasn't

{fetta-0.1.2 → fetta-0.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fetta"
-version = "0.1.2"
+version = "0.2.0"
 description = "Tool that fetches web pages for assistants"
 readme = "README.md"
 license = "MIT"
@@ -14,7 +14,7 @@ requires-python = ">=3.12"
 dependencies = [
     "beautifulsoup4>=4.15.0,<5",
     "playwright==1.63.0",
-    "requests>=2.34.2,<3",
+    "httpx>=0.28.1,<1",
 ]
 
 [[project.authors]]
@@ -39,7 +39,7 @@ dev = [
     "jupyterlab>=4.6.4",
     "pytest>=9.1.1",
     "pytest-asyncio>=1.4.0",
-    "requests-mock>=1.12.1",
+    "respx>=0.23.1,<1",
     "ruff>=0.16.8",
 ]
 
@@ -50,6 +50,8 @@ module-root = "src"
 asyncio_mode = "auto"
 asyncio_default_fixture_loop_scope = "session"
 asyncio_default_test_loop_scope = "session"
+markers = ["integration: tests that hit the real network"]
+addopts = "-m 'not integration'"
 testpaths = ["tests"]
 
 [tool.ruff]

{fetta-0.1.2 → fetta-0.2.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "fetta"
-version = "0.1.2"
+version = "0.2.0"
 description = "Tool that fetches web pages for assistants"
 readme = "README.md"
 authors = [{ name = "Sergei" }]
@@ -15,7 +15,7 @@ requires-python = ">=3.12"
 dependencies = [
     "beautifulsoup4>=4.15.0,<5",
     "playwright==1.63.0",
-    "requests>=2.34.2,<3"
+    "httpx>=0.28.1,<1"
 ]
 
 [project.scripts]
@@ -37,7 +37,7 @@ dev = [
     "jupyterlab>=4.6.4",
     "pytest>=9.1.1",
     "pytest-asyncio>=1.4.0",
-    "requests-mock>=1.12.1",
+    "respx>=0.23.1,<1",
     "ruff>=0.16.8",
 ]
 
@@ -48,6 +48,8 @@ module-root = "src"
 asyncio_mode = "auto"
 asyncio_default_fixture_loop_scope = "session"
 asyncio_default_test_loop_scope = "session"
+markers = ["integration: tests that hit the real network"]
+addopts = "-m 'not integration'"
 testpaths = ["tests"]
 
 [tool.ruff]

fetta-0.2.0/src/fetta/__init__.py

@@ -0,0 +1,18 @@
+from .browser import BrowserManager
+from .fetchers import BrowserFetcher, HttpFetcher
+from .json_fetcher import JsonFetcher
+from .models import Link, Image, PageContent, PageMetadata, FetchResult
+from .protocol import FetcherProtocol
+
+__all__ = [
+    "BrowserManager",
+    "PageContent",
+    "Link",
+    "Image",
+    "PageMetadata",
+    "FetchResult",
+    "BrowserFetcher",
+    "HttpFetcher",
+    "FetcherProtocol",
+    "JsonFetcher",
+]

fetta-0.2.0/src/fetta/base.py

@@ -0,0 +1,75 @@
+from abc import ABC, abstractmethod
+from typing import Any
+
+from .models import FetchResult, PageContent
+from .page_parser.constants import DEFAULT_SECTIONS
+from .page_parser.parser import parse_html
+
+
+class BaseFetcher(ABC):
+    """
+    Abstract base class for fetchers.
+    """
+
+    @abstractmethod
+    async def fetch(self, url: str, **kwargs: Any) -> FetchResult:
+        """Fetch a page and return its parsed content."""
+        pass
+
+    @abstractmethod
+    async def close(self) -> None:
+        """Release resources (HTTP connections, browser context, etc.)."""
+        pass
+
+    async def __aenter__(self) -> "BaseFetcher":
+        return self
+
+    async def __aexit__(self, exc_type, exc, tb) -> None:
+        await self.close()
+
+    def _parse(
+        self,
+        html: str,
+        url: str,
+        status: int,
+        title: str = "",
+        csr_data: list[dict] = None,
+        include: tuple[str, ...] = DEFAULT_SECTIONS
+    ) -> FetchResult:
+        """
+        Turn raw HTML into a PageContent.
+
+        Extracts the title (if not already provided), strips boilerplate
+        tags, and normalizes the remaining text.
+
+        Args:
+            html: Raw HTML as returned by the transport.
+            url: Original URL (stored on the result for traceability).
+            status: HTTP status code.
+            title: Pre-extracted title. If empty, the <title> tag is used.
+        """
+        parsed_content = parse_html(
+            html=html,
+            page_url=url,
+            include=include
+        )
+        page_metadata = parsed_content.get("metadata") or {}
+
+        return FetchResult(
+            url=url,
+            content=PageContent(
+                html=parsed_content.get("html") or "",
+                text=parsed_content.get("text"),
+                raw_html=parsed_content.get("raw_html"),
+                raw_text=parsed_content.get("raw_text"),
+                metadata=page_metadata,
+                links=tuple(parsed_content.get("links") or []),
+                images=parsed_content.get("images") or [],
+                csr_data=tuple(csr_data) if csr_data is not None else None
+            ),
+            title=page_metadata.get("title"),
+            icon=page_metadata.get("icon"),
+            image=page_metadata.get("image"),
+            description=page_metadata.get("description"),
+            status=status
+        )

{fetta-0.1.2 → fetta-0.2.0}/src/fetta/fetchers.py

@@ -1,11 +1,11 @@
 from typing import Any
 
-from fetta.protocol import FetcherProtocol
-from fetta.base import BaseFetcher
-from fetta.browser import BrowserManager
-from fetta.models import PageContent
-from fetta.playwright_fetcher import PlaywrightFetcher
-from fetta.requests_fetcher import RequestsFetcher
+from .httpx_fetcher import HttpxFetcher
+from .protocol import FetcherProtocol
+from .base import BaseFetcher
+from .browser import BrowserManager
+from .models import FetchResult
+from .playwright_fetcher import PlaywrightFetcher
 
 
 class HttpFetcher(BaseFetcher):
@@ -13,7 +13,7 @@ class HttpFetcher(BaseFetcher):
     Fetches pages over plain HTTP (no browser, no JavaScript).
 
     A facade over pluggable HTTP engines. By default, it uses
-    RequestsFetcher (sync `requests` offloaded to a worker thread).
+    HttpxFetcher (sync `requests` offloaded to a worker thread).
     """
 
     def __init__(
@@ -21,9 +21,9 @@ class HttpFetcher(BaseFetcher):
         *,
         engine: FetcherProtocol | None = None,
     ) -> None:
-        self._engine: FetcherProtocol = engine or RequestsFetcher()
+        self._engine: FetcherProtocol = engine or HttpxFetcher()
 
-    async def fetch(self, url: str, **kwargs: Any) -> PageContent:
+    async def fetch(self, url: str, **kwargs: Any) -> FetchResult:
         return await self._engine.fetch(url, **kwargs)
 
     async def close(self) -> None:
@@ -42,11 +42,11 @@ class BrowserFetcher(BaseFetcher):
         self,
         manager: BrowserManager,
         *,
-        engine: FetcherProtocol | None = None,
+        engine: FetcherProtocol | None = None
     ) -> None:
         self._engine: FetcherProtocol = engine or PlaywrightFetcher(manager)
 
-    async def fetch(self, url: str, **kwargs: Any) -> PageContent:
+    async def fetch(self, url: str, **kwargs: Any) -> FetchResult:
         return await self._engine.fetch(url, **kwargs)
 
     async def close(self) -> None:

fetta-0.2.0/src/fetta/httpx_fetcher.py

@@ -0,0 +1,67 @@
+from typing import Any
+
+import httpx
+
+from .models import FetchResult
+from .base import BaseFetcher
+from .page_parser.constants import DEFAULT_SECTIONS
+
+
+class HttpxFetcher(BaseFetcher):
+    """
+    Native async HTTP fetcher using httpx.AsyncClient.
+
+    Connection pooling is handled by the single client instance; do not
+    create a new client per request (that defeats pooling and adds
+    TLS-handshake overhead).
+    """
+
+    def __init__(self, *, timeout: float = 10.0, http2: bool = False) -> None:
+        self._client = httpx.AsyncClient(
+            timeout=timeout,
+            follow_redirects=True,
+            http2=http2,
+            headers={
+                "User-Agent": (
+                    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                    "AppleWebKit/537.36 (KHTML, like Gecko) "
+                    "Chrome/149.0.0.0 Safari/537.36"
+                ),
+                "Accept": (
+                    "text/html,application/xhtml+xml,application/xml;q=0.9,"
+                    "image/avif,image/webp,*/*;q=0.8"
+                ),
+                "Accept-Language": "en-GB,en;q=0.9,en-US;q=0.8,en;q=0.7",
+                # Let httpx negotiate Accept-Encoding automatically
+                "Connection": "keep-alive",
+                "Upgrade-Insecure-Requests": "1",
+                "Sec-Fetch-Dest": "document",
+                "Sec-Fetch-Mode": "navigate",
+                "Sec-Fetch-Site": "none",
+                "Sec-Fetch-User": "?1",
+                "Cache-Control": "max-age=0",
+                "sec-ch-ua": '"Chromium";v="149", "Not)A;Brand";v="24"',
+                "sec-ch-ua-mobile": "?0",
+                "sec-ch-ua-platform": '"Windows"',
+            },
+        )
+
+    async def fetch(
+        self,
+        url: str,
+        *,
+        include: tuple[str, ...] = DEFAULT_SECTIONS,
+        **kwargs: Any
+    ) -> FetchResult:
+        response = await self._client.get(url)
+        response.raise_for_status()
+
+        return self._parse(
+            html=response.text,
+            url=url,
+            status=response.status_code,
+            include=include,
+        )
+
+    async def close(self) -> None:
+        await self._client.aclose()

{fetta-0.1.2 → fetta-0.2.0}/src/fetta/json_fetcher.py

@@ -1,6 +1,6 @@
 from fetta.base import BaseFetcher
 from fetta.browser import BrowserManager
-from fetta.models import PageContent
+from fetta.models import FetchResult
 
 
 _BLOCK = {"image", "media", "font"}
@@ -16,8 +16,9 @@ class JsonFetcher(BaseFetcher):
         *,
         wait_until: str = "load",
         timeout_ms: int = 30_000,
+        include: tuple[str, ...] = ("metadata",),
         **kwargs,
-    ) -> PageContent:
+    ) -> FetchResult:
 
         json_responses: list[dict] = []
 
@@ -50,7 +51,14 @@ class JsonFetcher(BaseFetcher):
             html = await page.content()
             title = await page.title()
 
-        return self._parse(html, url, response.status if response else 0, title, json_responses)
+        return self._parse(
+            html=html,
+            url=url,
+            status=response.status if response else 0,
+            title=title,
+            csr_data=json_responses,
+            include=include
+        )
 
     async def close(self) -> None:
         # Browser lifecycle is owned by BrowserManager, not by the fetcher.

fetta-0.2.0/src/fetta/models.py

@@ -0,0 +1,54 @@
+from dataclasses import dataclass
+from typing import TypedDict
+
+
+class PageMetadata(TypedDict):
+    page_url: str
+    canonical_url: str | None
+    icon: str | None
+    title: str | None
+    description: str | None
+    site_name: str | None
+    og_image: str | None
+    og_type: str | None
+    ld_json: tuple | None
+
+
+@dataclass(frozen=True, slots=True)
+class Link:
+    text: str
+    url: str
+    external: bool
+    section: str | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class Image:
+    src: str
+    alt: str | None = None
+    title: str | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class PageContent:
+    html: str
+    text: str | None
+    raw_html: str | None
+    raw_text: str | None
+    metadata: PageMetadata
+    links: tuple[Link, ...]
+    images: tuple[Image, ...]
+    csr_data: tuple[dict, ...] | None
+
+
+@dataclass(frozen=True, slots=True)
+class FetchResult:
+    url: str
+    status: int
+    # Meta
+    title: str | None
+    icon: str | None
+    description: str | None
+    image: str | None
+    # Content
+    content: PageContent

fetta-0.2.0/src/fetta/page_parser/cleaning.py

@@ -0,0 +1,76 @@
+import copy
+import re
+
+from bs4 import Comment
+from bs4.element import Tag
+
+from .constants import DROP_TAGS, NOISE_RE, KEEP_TAGS, KEEP_ATTRS
+
+
+STRUCTURAL = frozenset({"html", "head", "body", "main", "article"})
+
+
+def clean_html(node: Tag, *, inplace: bool = False) -> Tag:
+    """Drop scripts, styles, comments, and noise blocks.
+
+    Returns a new soup unless inplace=True
+    """
+
+    if not inplace:
+        node = copy.deepcopy(node)
+
+    for tag in node(DROP_TAGS):
+        if not tag.decomposed:
+            tag.decompose()
+
+    for comment in node.find_all(string=lambda s: isinstance(s, Comment)):
+        comment.extract()
+
+    for tag in node.find_all(True):
+        if tag.name in STRUCTURAL or tag.decomposed or not tag.attrs:
+            continue
+
+        attrs = " ".join(
+            filter(
+                None,
+                (
+                    " ".join(tag.get("class", [])),
+                    tag.get("id", ""),
+                    tag.get("role", ""),
+                ),
+            )
+        )
+        if attrs and NOISE_RE.search(attrs):
+            tag.decompose()
+
+    return node
+
+
+def minify_html(
+    node: Tag, *, max_chars: int = 25_000, inplace: bool = False
+) -> tuple[str, str | None]:
+    """Strip unwanted tags/attrs and collapse whitespace.
+
+    Returns a string. Input tree is copied unless inplace=True
+    """
+    if not inplace:
+        node = copy.deepcopy(node)
+
+    for tag in list(node.find_all(True)):
+        if tag.name not in KEEP_TAGS:
+            tag.unwrap()
+            continue
+
+        for attr in list(tag.attrs):
+            if attr not in KEEP_ATTRS or tag.attrs[attr] in ("", None):
+                del tag[attr]
+
+    # Flatten bare wrappers
+    for tag in node.find_all(["div", "span"]):
+        if not tag.attrs:
+            tag.unwrap()
+
+    text = str(node)
+    text = re.sub(r"\s+", " ", text)
+    text = re.sub(r">\s+<", "><", text)
+    return text.strip()[:max_chars], node.get_text(" ", strip=True)[:max_chars]

fetta-0.2.0/src/fetta/page_parser/constants.py

@@ -0,0 +1,61 @@
+import re
+
+DROP_TAGS = frozenset({
+    "script", "style", "noscript", "template",
+    "iframe", "canvas", "svg", "object", "embed",
+    "form", "input", "select", "textarea", "button", "label",
+    "video", "audio", "source", "track",
+    "dialog", "menu"
+})
+
+NOISE_RE = re.compile(
+    r"(cookie|consent|gdpr|"
+    r"banner|advert|ads?[-_]|adslot|promo|sponsor|"
+    r"popup|modal|overlay|lightbox|tooltip|"
+    r"social|share|sharing|"
+    r"newsletter|subscribe|subscription|"
+    r"comment|disqus|"
+    r"sidebar|aside|widget|"
+    r"breadcrumb|pagination|pager|"
+    r"^nav$|navigation|menu|toolbar|"
+    r"^footer$|^header$|"
+    r"related|recommend|similar|"
+    r"tag-?list|tags-?list|"
+    r"author-?box|author-?card)",
+    re.IGNORECASE
+)
+
+KEEP_ATTRS = frozenset({"href", "src", "alt", "title", "datetime"})
+
+KEEP_TAGS = frozenset({
+    "h1","h2","h3","h4","h5","h6",
+    "p","br","hr","ul","ol","li",
+    "a","img","figure","figcaption",
+    "blockquote","strong","em","b","i",
+    "table","thead","tbody","tr","th","td",
+    "time","span","div"
+})
+
+NAV_CONTAINERS = (
+    "nav",
+    "header",
+    "footer",
+    '[role="navigation"]',
+    '[class*="nav" i]',
+    '[class*="menu" i]',
+    '[class*="breadcrumb" i]',
+    '[class*="footer" i]',
+    '[class*="header" i]'
+)
+NAV_HEADER_SELECTOR = ("header nav", "header", '[class*="header" i] nav', '[class*="header" i]')
+NAV_FOOTER_SELECTOR = ("footer nav", "footer", '[class*="footer" i]')
+
+SKIP_SCHEMES = ("javascript:", "mailto:", "tel:", "data:", "#")
+
+SKIP_PATH_PARTS = (
+    "/login", "/logout", "/signup", "/register", "/cart",
+    "/account", "/privacy", "/terms", "/cookie"
+)
+
+DEFAULT_SECTIONS = ("metadata", "links", "images", "html", "text")
+VALID_SECTIONS = frozenset(DEFAULT_SECTIONS + ("raw_html", "raw_text"))

fetta-0.2.0/src/fetta/page_parser/content.py

@@ -0,0 +1,37 @@
+from bs4 import BeautifulSoup
+from bs4.element import Tag
+
+
+def _text_len(node: Tag) -> int:
+    return len(node.get_text(" ", strip=True))
+
+
+def _link_density(node: Tag) -> float:
+    total = _text_len(node)
+    if total == 0:
+        return 1.0
+    link_text = sum(_text_len(a) for a in node.find_all("a"))
+    return link_text / total
+
+
+def _score(node: Tag) -> float:
+    length = _text_len(node)
+    if length < 100:
+        return 0.0
+    return length * (1.0 - _link_density(node))
+
+
+def extract_main_content(soup: BeautifulSoup) -> Tag:
+    """Return the main content node of the page. Does not mutate soup."""
+    for selector in ("article", "main", "[role=main]"):
+        node = soup.select_one(selector)
+        if node and _text_len(node) >= 200:
+            return node
+
+    best, best_score = None, 0.0
+    for node in soup.find_all(["div", "section", "td"]):
+        s = _score(node)
+        if s > best_score:
+            best, best_score = node, s
+
+    return best if best is not None else (soup.body or soup)

fetta-0.2.0/src/fetta/page_parser/helpers.py

@@ -0,0 +1,24 @@
+from urllib.parse import urljoin, urlparse
+from bs4 import BeautifulSoup
+
+
+def canonical_url(soup: BeautifulSoup, page_url: str | None = None) -> str | None:
+    tag = soup.find("link", rel="canonical")
+    href = tag.get("href") if tag else None
+    if href and page_url:
+        return urljoin(page_url, href)
+    return href or None
+
+
+def base_url(soup: BeautifulSoup, page_url: str | None = None) -> str:
+    return page_url or canonical_url(soup, page_url) or ""
+
+
+def resolve(href: str, base: str) -> tuple[str, str]:
+    """Return (absolute_url, netloc)."""
+    abs_url = urljoin(base, href)
+    return abs_url, urlparse(abs_url).netloc
+
+
+def is_http(url: str) -> bool:
+    return urlparse(url).scheme in ("http", "https")

fetta-0.2.0/src/fetta/page_parser/images.py

@@ -0,0 +1,20 @@
+from bs4 import BeautifulSoup
+from .helpers import base_url, resolve
+from ..models import Image
+
+
+def extract_images(soup: BeautifulSoup, page_url: str | None = None) -> list[dict]:
+    base = base_url(soup, page_url)
+    images = []
+    for img in soup.find_all("img"):
+        src = img.get("src") or img.get("data-src") or img.get("data-lazy-src")
+        if not src:
+            continue
+        images.append(
+            Image(
+                src=resolve(src, base)[0],
+                alt=img.get("alt") or None,
+                title=img.get("title") or None
+            )
+        )
+    return images

fetta-0.2.0/src/fetta/page_parser/metadata.py

@@ -0,0 +1,48 @@
+import json
+from urllib.parse import urlparse, urljoin
+from bs4 import BeautifulSoup
+from .helpers import base_url, canonical_url
+from ..models import PageMetadata
+
+
+def extract_favicon(soup: BeautifulSoup, page_url: str | None = None) -> str | None:
+    """Return absolute URL of the page's favicon, or None."""
+    for link in soup.find_all("link", href=True):
+        rels = link.get("rel") or []
+        rels = [r.lower() for r in rels]
+        if "icon" in rels:
+            return urljoin(page_url or "", link["href"].strip())
+
+    # Fallback: the browser default location
+    return urljoin(page_url, "/favicon.ico") if page_url else None
+
+
+def extract_metadata(soup: BeautifulSoup, page_url: str | None = None) -> PageMetadata:
+
+    def meta(*, prop: str | None = None, name: str | None = None) -> str | None:
+        attrs = {k: v for k, v in (("property", prop), ("name", name)) if v}
+        tag = soup.find("meta", attrs=attrs)
+        return tag["content"] if tag and tag.get("content") else None
+
+    canonical = canonical_url(soup, page_url)
+    base = base_url(soup, page_url)
+
+    ld_objects = []
+    for script in soup.find_all("script", type="application/ld+json"):
+        try:
+            data = json.loads(script.string or "{}")
+        except json.JSONDecodeError:
+            continue
+        ld_objects.extend(data if isinstance(data, list) else [data])
+
+    return PageMetadata(
+        page_url=base,
+        canonical_url=canonical,
+        icon=extract_favicon(soup, page_url),
+        title=meta(prop="og:title") or (soup.title.string if soup.title else None),
+        description=meta(prop="og:description") or meta(name="description"),
+        site_name=meta(prop="og:site_name") or urlparse(base).netloc or None,
+        og_image=meta(prop="og:image"),
+        og_type=meta(prop="og:type"),
+        ld_json=tuple(ld_objects)
+    )

fetta-0.2.0/src/fetta/page_parser/navigation.py

@@ -0,0 +1,54 @@
+from bs4 import BeautifulSoup
+from .constants import (
+    NAV_CONTAINERS,
+    SKIP_SCHEMES,
+    SKIP_PATH_PARTS,
+    NAV_HEADER_SELECTOR,
+    NAV_FOOTER_SELECTOR,
+)
+from .helpers import base_url, is_http, resolve
+from ..models import Link
+
+
+def extract_navigation(soup: BeautifulSoup, page_url: str | None = None) -> list:
+    base = base_url(soup, page_url)
+    base_host = resolve(base, base)[1]
+    links, seen = [], set()
+
+    def collect(container, section):
+        for a in container.find_all("a", href=True):
+            href = a["href"].strip()
+            if not href or href.startswith(SKIP_SCHEMES):
+                continue
+            abs_url, host = resolve(href, base)
+            if not is_http(abs_url):
+                continue
+            if any(p in abs_url for p in SKIP_PATH_PARTS):
+                continue
+            if abs_url in seen:
+                continue
+            text = a.get_text(" ", strip=True)
+            if not text:
+                continue
+            seen.add(abs_url)
+            links.append(Link(text=text, url=abs_url, external=host != base_host, section=section))
+
+    def pick(selectors, section):
+        for sel in selectors:
+            node = soup.select_one(sel)
+            if node:
+                collect(node, section)
+                break
+
+    breadcrumb_node = (
+            soup.select_one('[class*="breadcrumb" i]')
+            or soup.select_one('nav[aria-label*="breadcrumb" i]')
+    )
+    if breadcrumb_node:
+        collect(breadcrumb_node, "breadcrumbs")
+
+    pick(NAV_HEADER_SELECTOR, "header")
+    pick(NAV_FOOTER_SELECTOR, "footer")
+    pick(NAV_CONTAINERS, "nav")
+
+    return links

fetta-0.2.0/src/fetta/page_parser/parser.py

@@ -0,0 +1,53 @@
+from bs4 import BeautifulSoup
+
+from .cleaning import clean_html, minify_html
+from .constants import DEFAULT_SECTIONS, VALID_SECTIONS
+from .content import extract_main_content
+from .images import extract_images
+from .metadata import extract_metadata
+from .navigation import extract_navigation
+
+
+def parse_html(
+    html: str,
+    page_url: str | None = None,
+    *,
+    include: tuple[str, ...] = DEFAULT_SECTIONS
+) -> dict:
+    """Parse an HTML page into structured sections.
+
+    include may contain any of: metadata, links, images, html, text,
+    raw_html, raw_text. Unknown values raise ValueError.
+    """
+    unknown = set(include) - VALID_SECTIONS
+    if unknown:
+        raise ValueError(f"Unknown sections: {sorted(unknown)}")
+
+    raw_soup = BeautifulSoup(html, "html.parser")
+    output: dict = {}
+
+    # Sections that must run on the unmodified DOM
+    if "metadata" in include:
+        output["metadata"] = extract_metadata(raw_soup, page_url)
+    if "links" in include:
+        output["links"] = extract_navigation(raw_soup, page_url)
+    if "images" in include:
+        output["images"] = extract_images(raw_soup, page_url)
+    if "raw_html" in include:
+        output["raw_html"] = html
+    if "raw_text" in include:
+        output["raw_text"] = raw_soup.get_text("\n", strip=True)
+
+    # Content runs on a cleaned copy. clean_html copies by default, so
+    # node stays intact for any later stage
+    output["html"] = ""
+    if "html" in include or "text" in include:
+        node = extract_main_content(raw_soup)
+        cleaned_node = clean_html(node)
+        minified_html, text = minify_html(cleaned_node, inplace=True)
+        if "html" in include:
+            output["html"] = minified_html
+        if "text" in include:
+            output["text"] = text
+
+    return output

{fetta-0.1.2 → fetta-0.2.0}/src/fetta/playwright_fetcher.py

@@ -1,7 +1,7 @@
 from fetta.base import BaseFetcher
 from fetta.browser import BrowserManager
-from fetta.models import PageContent
-
+from fetta.models import FetchResult
+from fetta.page_parser.constants import DEFAULT_SECTIONS
 
 _BLOCK = {"image", "media", "font"}
 
@@ -17,8 +17,9 @@ class PlaywrightFetcher(BaseFetcher):
         wait_for: str | None = None,
         wait_until: str = "load",
         timeout_ms: int = 30_000,
+        include: tuple[str, ...] = DEFAULT_SECTIONS,
         **kwargs,
-    ) -> PageContent:
+    ) -> FetchResult:
         async with self._browser.page() as page:
             await page.route("**/*", self._block_resources)
 
@@ -41,7 +42,13 @@ class PlaywrightFetcher(BaseFetcher):
             html = await page.content()
             title = await page.title()
 
-        return self._parse(html, url, response.status if response else 0, title)
+        return self._parse(
+            html=html,
+            url=url,
+            status=response.status if response else 0,
+            title=title,
+            include=include
+        )
 
     async def close(self) -> None:
         # Browser lifecycle is owned by BrowserManager, not by the fetcher.

{fetta-0.1.2 → fetta-0.2.0}/src/fetta/protocol.py

@@ -1,8 +1,8 @@
 from typing import Protocol
 
-from fetta.models import PageContent
+from fetta.models import FetchResult
 
 
 class FetcherProtocol(Protocol):
-    async def fetch(self, url: str, **kwargs) -> PageContent: ...
+    async def fetch(self, url: str, **kwargs) -> FetchResult: ...
     async def close(self) -> None: ...

fetta-0.1.2/src/fetta/__init__.py

@@ -1,8 +0,0 @@
-from fetta.browser import BrowserManager
-from fetta.fetchers import BrowserFetcher, HttpFetcher
-from fetta.json_fetcher import JsonFetcher
-from fetta.models import PageContent
-from fetta.protocol import FetcherProtocol
-from fetta.requests_fetcher import RequestsFetcher
-
-__all__ = ["BrowserManager", "PageContent", "BrowserFetcher", "HttpFetcher", "FetcherProtocol"]

fetta-0.1.2/src/fetta/base.py

@@ -1,61 +0,0 @@
-from abc import ABC, abstractmethod
-from typing import Any
-
-from bs4 import BeautifulSoup
-
-from fetta.models import PageContent
-
-# Tags that never contain useful article text.
-# Kept module-level so subclasses can reference the same tuple.
-_STRIP_TAGS = ("script", "style", "nav", "footer", "header", "noscript")
-
-
-class BaseFetcher(ABC):
-    """
-    Abstract base class for fetchers.
-    """
-
-    @abstractmethod
-    async def fetch(self, url: str, **kwargs: Any) -> PageContent:
-        """Fetch a page and return its parsed content."""
-        pass
-
-    @abstractmethod
-    async def close(self) -> None:
-        """Release resources (HTTP connections, browser context, etc.)."""
-        pass
-
-    async def __aenter__(self) -> "BaseFetcher":
-        return self
-
-    async def __aexit__(self, exc_type, exc, tb) -> None:
-        await self.close()
-
-    def _parse(
-        self, html: str, url: str, status: int, title: str = "", data: list[dict] = None
-    ) -> PageContent:
-        """
-        Turn raw HTML into a PageContent.
-
-        Extracts the title (if not already provided), strips boilerplate
-        tags, and normalizes the remaining text.
-
-        Args:
-            html: Raw HTML as returned by the transport.
-            url: Original URL (stored on the result for traceability).
-            status: HTTP status code.
-            title: Pre-extracted title. If empty, the <title> tag is used.
-        """
-        soup = BeautifulSoup(html, "html.parser")
-
-        # Extract title before stripping tags as some sites put <style>
-        # inside <head>, and decomposing early keeps the tree clean.
-        if not title and soup.title is not None:
-            title = soup.title.get_text(strip=True)
-
-        for tag in soup(_STRIP_TAGS):
-            tag.decompose()
-
-        text = soup.get_text(separator="\n", strip=True)
-
-        return PageContent(url=url, title=title, text=text, html=html, data=data, status=status)

fetta-0.1.2/src/fetta/models.py

@@ -1,11 +0,0 @@
-from dataclasses import dataclass
-
-
-@dataclass(frozen=True, slots=True)
-class PageContent:
-    url: str
-    title: str
-    text: str
-    html: str
-    data: list[dict] | None
-    status: int

fetta-0.1.2/src/fetta/requests_fetcher.py

@@ -1,53 +0,0 @@
-import asyncio
-from typing import Any
-
-import requests
-
-from fetta.base import BaseFetcher
-from fetta.models import PageContent
-
-
-class RequestsFetcher(BaseFetcher):
-    """
-    Synchronous requests, offloaded to worker threads.
-    """
-
-    def __init__(self, *, timeout: float = 10.0) -> None:
-        self._session = requests.Session()
-        self._session.headers.update(
-            {
-                "User-Agent": (
-                    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-                    "AppleWebKit/537.36 (KHTML, like Gecko) "
-                    "Chrome/149.0.0.0 Safari/537.36"
-                ),
-                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-                "Accept-Language": "en-GB,en;q=0.9,en-US;q=0.8,en;q=0.7",
-                "Accept-Encoding": "gzip, deflate, br",
-                "Connection": "keep-alive",
-                "Upgrade-Insecure-Requests": "1",
-                "Sec-Fetch-Dest": "document",
-                "Sec-Fetch-Mode": "navigate",
-                "Sec-Fetch-Site": "none",
-                "Sec-Fetch-User": "?1",
-                "Cache-Control": "max-age=0",
-                "sec-ch-ua": '"Chromium";v="149", "Not)A;Brand";v="24"',
-                "sec-ch-ua-mobile": "?0",
-                "sec-ch-ua-platform": '"Windows"',
-            }
-        )
-        self._timeout = timeout
-
-    def _get_sync(self, url: str) -> tuple[str, int]:
-        response = self._session.get(url, timeout=self._timeout)
-        response.raise_for_status()
-        return response.text, response.status_code
-
-    async def fetch(self, url: str, **kwargs: Any) -> PageContent:
-        # Hand the blocking call to a thread. The event loop is free
-        # to handle other requests while this thread waits on the socket.
-        html, status = await asyncio.to_thread(self._get_sync, url)
-        return self._parse(html, url, status)
-
-    async def close(self) -> None:
-        self._session.close()