fetta 0.2.0__tar.gz → 0.3.0__tar.gz

{fetta-0.2.0 → fetta-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fetta
-Version: 0.2.0
+Version: 0.3.0
 Summary: Tool that fetches web pages for assistants
 Author: Sergei
 License-Expression: MIT

{fetta-0.2.0 → fetta-0.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fetta"
-version = "0.2.0"
+version = "0.3.0"
 description = "Tool that fetches web pages for assistants"
 readme = "README.md"
 license = "MIT"

{fetta-0.2.0 → fetta-0.3.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "fetta"
-version = "0.2.0"
+version = "0.3.0"
 description = "Tool that fetches web pages for assistants"
 readme = "README.md"
 authors = [{ name = "Sergei" }]

{fetta-0.2.0 → fetta-0.3.0}/src/fetta/__init__.py

@@ -1,5 +1,5 @@
 from .browser import BrowserManager
-from .fetchers import BrowserFetcher, HttpFetcher
+from .fetchers import BrowserFetcher, HttpFetcher, SmartFetcher
 from .json_fetcher import JsonFetcher
 from .models import Link, Image, PageContent, PageMetadata, FetchResult
 from .protocol import FetcherProtocol
@@ -15,4 +15,5 @@ __all__ = [
     "HttpFetcher",
     "FetcherProtocol",
     "JsonFetcher",
+    "SmartFetcher"
 ]

{fetta-0.2.0 → fetta-0.3.0}/src/fetta/base.py

@@ -1,7 +1,7 @@
 from abc import ABC, abstractmethod
 from typing import Any
 
-from .models import FetchResult, PageContent
+from .models import FetchResult, PageContent, PageMetadata, FetcherKind
 from .page_parser.constants import DEFAULT_SECTIONS
 from .page_parser.parser import parse_html
 
@@ -11,6 +11,8 @@ class BaseFetcher(ABC):
     Abstract base class for fetchers.
     """
 
+    fetcher_kind: FetcherKind
+
     @abstractmethod
     async def fetch(self, url: str, **kwargs: Any) -> FetchResult:
         """Fetch a page and return its parsed content."""
@@ -62,7 +64,7 @@ class BaseFetcher(ABC):
                 text=parsed_content.get("text"),
                 raw_html=parsed_content.get("raw_html"),
                 raw_text=parsed_content.get("raw_text"),
-                metadata=page_metadata,
+                metadata=parsed_content.get("metadata"),
                 links=tuple(parsed_content.get("links") or []),
                 images=parsed_content.get("images") or [],
                 csr_data=tuple(csr_data) if csr_data is not None else None
@@ -71,5 +73,6 @@ class BaseFetcher(ABC):
             icon=page_metadata.get("icon"),
             image=page_metadata.get("image"),
             description=page_metadata.get("description"),
-            status=status
+            status=status,
+            fetcher_kind=self.fetcher_kind
         )

fetta-0.3.0/src/fetta/browser.py

@@ -0,0 +1,54 @@
+from asyncio import Semaphore
+from contextlib import asynccontextmanager
+from typing import AsyncIterator
+
+from playwright.async_api import (
+    Browser,
+    Playwright,
+    async_playwright,
+    ViewportSize,
+    Page
+)
+
+
+class BrowserManager:
+    def __init__(self, max_contexts: int = 10) -> None:
+        self._semaphore = Semaphore(max_contexts)
+        self._playwright: Playwright | None = None
+        self._browser: Browser | None = None
+
+    async def start(self) -> None:
+        self._playwright = await async_playwright().start()
+        self._browser = await self._playwright.chromium.launch(headless=True)
+
+    async def stop(self) -> None:
+        if self._browser is not None:
+            await self._browser.close()
+            self._browser = None
+        if self._playwright is not None:
+            await self._playwright.stop()
+            self._playwright = None
+
+    @asynccontextmanager
+    async def page(self) -> AsyncIterator[Page]:
+        if self._browser is None:
+            raise RuntimeError("BrowserManager is not started")
+        async with self._semaphore:
+            context = await self._browser.new_context(
+                user_agent=(
+                    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                    "AppleWebKit/537.36 (KHTML, like Gecko) "
+                    "Chrome/153.0.0.0 Safari/537.36"
+                ),
+                viewport=ViewportSize(**{"width": 1920, "height": 1080}),
+                locale="en-GB",
+                extra_http_headers={
+                    "Accept-Language": "en-GB,en;q=0.9,en-US;q=0.8,en;q=0.7",
+                },
+            )
+            page = await context.new_page()
+            try:
+                yield page
+            finally:
+                await context.close()
+

{fetta-0.2.0 → fetta-0.3.0}/src/fetta/fetchers.py

@@ -4,8 +4,9 @@ from .httpx_fetcher import HttpxFetcher
 from .protocol import FetcherProtocol
 from .base import BaseFetcher
 from .browser import BrowserManager
-from .models import FetchResult
+from .models import FetchResult, FetcherKind
 from .playwright_fetcher import PlaywrightFetcher
+from .smart_fetcher import VanillaSmartFetcher
 
 
 class HttpFetcher(BaseFetcher):
@@ -16,6 +17,8 @@ class HttpFetcher(BaseFetcher):
     HttpxFetcher (sync `requests` offloaded to a worker thread).
     """
 
+    fetcher_kind: FetcherKind = "http"
+
     def __init__(
         self,
         *,
@@ -38,6 +41,8 @@ class BrowserFetcher(BaseFetcher):
     used; alternative engines can be injected via the `engine` argument.
     """
 
+    fetcher_kind: FetcherKind = "browser"
+
     def __init__(
         self,
         manager: BrowserManager,
@@ -51,3 +56,36 @@ class BrowserFetcher(BaseFetcher):
 
     async def close(self) -> None:
         await self._engine.close()
+
+
+
+class SmartFetcher(BaseFetcher):
+    """
+    Fetches pages with the cheapest fetcher that can render them.
+
+    The page is first fetched via the HTML fetcher. If the response
+    looks client-side rendered, the browser fetcher is used instead.
+    """
+
+    def __init__(
+        self,
+        manager: BrowserManager | None = None,
+        *,
+        html_fetcher: FetcherProtocol | None = None,
+        browser_fetcher: FetcherProtocol | None = None,
+    ) -> None:
+        html_engine: FetcherProtocol = html_fetcher or HttpxFetcher()
+        if not browser_fetcher and not manager:
+            browser_engine = None
+        else:
+            browser_engine = browser_fetcher or PlaywrightFetcher(manager)
+        self._engine = VanillaSmartFetcher(
+            html_fetcher=html_engine,
+            browser_fetcher=browser_engine
+        )
+
+    async def fetch(self, url: str, **kwargs: Any) -> FetchResult:
+        return await self._engine.fetch(url, **kwargs)
+
+    async def close(self) -> None:
+        await self._engine.close()

{fetta-0.2.0 → fetta-0.3.0}/src/fetta/httpx_fetcher.py

@@ -2,7 +2,7 @@ from typing import Any
 
 import httpx
 
-from .models import FetchResult
+from .models import FetchResult, FetcherKind
 from .base import BaseFetcher
 from .page_parser.constants import DEFAULT_SECTIONS
 
@@ -16,6 +16,8 @@ class HttpxFetcher(BaseFetcher):
     TLS-handshake overhead).
     """
 
+    fetcher_kind: FetcherKind = "http"
+
     def __init__(self, *, timeout: float = 10.0, http2: bool = False) -> None:
         self._client = httpx.AsyncClient(
             timeout=timeout,

{fetta-0.2.0 → fetta-0.3.0}/src/fetta/json_fetcher.py

@@ -1,12 +1,14 @@
-from fetta.base import BaseFetcher
-from fetta.browser import BrowserManager
-from fetta.models import FetchResult
-
+from .base import BaseFetcher
+from .browser import BrowserManager
+from .models import FetchResult, FetcherKind
 
 _BLOCK = {"image", "media", "font"}
 
 
 class JsonFetcher(BaseFetcher):
+
+    fetcher_kind: FetcherKind = "browser"
+
     def __init__(self, browser: BrowserManager) -> None:
         self._browser = browser
 

{fetta-0.2.0 → fetta-0.3.0}/src/fetta/models.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import TypedDict
+from typing import TypedDict, Literal
 
 
 class PageMetadata(TypedDict):
@@ -35,16 +35,19 @@ class PageContent:
     text: str | None
     raw_html: str | None
     raw_text: str | None
-    metadata: PageMetadata
+    metadata: PageMetadata | None
     links: tuple[Link, ...]
     images: tuple[Image, ...]
     csr_data: tuple[dict, ...] | None
 
 
+FetcherKind = Literal["http", "browser"]
+
 @dataclass(frozen=True, slots=True)
 class FetchResult:
     url: str
     status: int
+    fetcher_kind: FetcherKind
     # Meta
     title: str | None
     icon: str | None

{fetta-0.2.0 → fetta-0.3.0}/src/fetta/playwright_fetcher.py

@@ -1,12 +1,15 @@
 from fetta.base import BaseFetcher
 from fetta.browser import BrowserManager
-from fetta.models import FetchResult
+from fetta.models import FetchResult, FetcherKind
 from fetta.page_parser.constants import DEFAULT_SECTIONS
 
 _BLOCK = {"image", "media", "font"}
 
 
 class PlaywrightFetcher(BaseFetcher):
+
+    fetcher_kind: FetcherKind = "browser"
+
     def __init__(self, browser: BrowserManager) -> None:
         self._browser = browser
 
@@ -45,7 +48,7 @@ class PlaywrightFetcher(BaseFetcher):
         return self._parse(
             html=html,
             url=url,
-            status=response.status if response else 0,
+            status=response.status if response else 500,
             title=title,
             include=include
         )

fetta-0.3.0/src/fetta/protocol.py

@@ -0,0 +1,11 @@
+from typing import Protocol, Any
+
+from fetta.models import FetchResult
+
+
+class FetcherProtocol(Protocol):
+    async def fetch(
+            self,
+            url: str,
+            **kwargs: Any) -> FetchResult: ...
+    async def close(self) -> None: ...

fetta-0.3.0/src/fetta/smart_fetcher.py

@@ -0,0 +1,137 @@
+import logging
+from dataclasses import replace
+from typing import Any
+
+from bs4 import BeautifulSoup
+
+from .protocol import FetcherProtocol
+from .models import FetchResult
+from .base import BaseFetcher
+from .page_parser.constants import DEFAULT_SECTIONS
+
+logger = logging.getLogger(__name__)
+
+
+EMPTY_ROOT_IDS = frozenset({
+    "root", "app", "__next", "__nuxt", "___gatsby",
+    "react-root", "svelte", "main-app", "ember-app"
+})
+
+FRAMEWORK_MARKERS = (
+    "data-reactroot", "_reactRootContainer",
+    "__NEXT_DATA__", "__NUXT__", "__SVELTE__",
+    "___gatsby", "ng-version=", "ng-app=",
+    "window.__INITIAL_STATE__", "window.__APOLLO_STATE__",
+)
+
+SSR_MARKERS = (
+    'data-server-rendered="true"',   # Vue SSR
+    "data-reactroot",                # React SSR (hydrated)
+)
+
+NON_CONTAINER_TAGS = frozenset({"script", "style", "link", "meta"})
+
+
+class VanillaSmartFetcher(BaseFetcher):
+    """
+    Wraps an HTML fetcher and a browser fetcher.
+
+    Contract: the underlying HTML fetcher must be able to return
+    `raw_html`, otherwise `_is_client_side_rendering` cannot detect
+    client-side-rendered pages reliably. VanillaSmartFetcher adds
+    `raw_html` to `include` whenever the caller supplies one.
+
+    `fetcher_kind` on the result reflects the fetcher that actually
+    produced the content ("http" or "browser"), not "composite".
+    """
+
+    def __init__(
+            self,
+            html_fetcher: FetcherProtocol,
+            browser_fetcher: FetcherProtocol | None
+    ):
+        self._html_fetcher = html_fetcher
+        self._browser_fetcher = browser_fetcher
+
+    async def fetch(
+            self,
+            url: str,
+            *,
+            include: tuple[str, ...] = DEFAULT_SECTIONS,
+            **kwargs: Any
+    ) -> FetchResult:
+        # Note: We must return raw_html to check whether content is CSR
+        html_include = tuple(set(include + ("raw_html", "raw_text")))
+        response = await self._html_fetcher.fetch(url=url, include=html_include, **kwargs)
+
+        if response.status >= 400:
+            return self._build_response(response, include)
+
+        if self._browser_fetcher and self._is_client_side_rendering(response):
+            return await self._browser_fetcher.fetch(url=url, include=include, **kwargs)
+
+        return self._build_response(response, include)
+
+    async def close(self) -> None:
+        await self._html_fetcher.close()
+        if self._browser_fetcher:
+            await self._browser_fetcher.close()
+
+    def _build_response(
+            self,
+            response: FetchResult,
+            include: tuple[str, ...]
+    ) -> FetchResult:
+        new_content = replace(
+            response.content,
+            raw_html=response.content.raw_html if "raw_html" in include else None,
+            raw_text=response.content.raw_text if "raw_text" in include else None
+        )
+        return replace(response, content=new_content)
+
+    def _is_client_side_rendering(self, response: FetchResult) -> bool:
+
+        html = response.content.raw_html
+        html_len = len(html)
+        if not html or html_len < 300:
+            logger.debug("CSR check: skip, html_len=%d < 300", html_len)
+            return False
+
+        raw_text = response.content.raw_text
+        text_len = len(raw_text) if raw_text else 0
+
+        has_framework = any(m in html for m in FRAMEWORK_MARKERS)
+        has_ssr = any(m in html for m in SSR_MARKERS)
+
+        logger.debug(
+            "CSR check: html_len=%d text_len=%d framework=%s ssr=%s",
+            html_len, text_len, has_framework, has_ssr,
+        )
+
+        if has_framework and has_ssr:
+            logger.debug("CSR check: NOT CSR (framework + SSR marker)")
+            return False
+
+        # No framework marker. Extremely thin text on a big page
+        if text_len < 2000 and len(html) > 10000:
+            logger.debug("CSR check: CSR (thin text on large html)")
+            return True
+
+        # Last resort: explicit empty root
+        if text_len < 2000:
+            soup = BeautifulSoup(html, "html.parser")
+            empty_root = self._has_empty_spa_root(soup)
+            logger.debug("CSR check: empty_root=%s", empty_root)
+            return empty_root
+
+        logger.debug("CSR check: NOT CSR (default)")
+        return False
+
+    def _has_empty_spa_root(self, soup: BeautifulSoup) -> bool:
+        for elem in soup.find_all(id=True):
+            if elem.name in NON_CONTAINER_TAGS:
+                continue
+            if elem["id"].lower() in EMPTY_ROOT_IDS and not elem.get_text(strip=True):
+                return True
+        return False
+

fetta-0.2.0/src/fetta/browser.py

@@ -1,49 +0,0 @@
-from contextlib import asynccontextmanager
-
-from playwright.async_api import (
-    Browser,
-    Playwright,
-    async_playwright,
-    ViewportSize,
-)
-
-
-class BrowserManager:
-    def __init__(self) -> None:
-        self._playwright: Playwright | None = None
-        self._browser: Browser | None = None
-
-    async def start(self) -> None:
-        self._playwright = await async_playwright().start()
-        self._browser = await self._playwright.chromium.launch(headless=True)
-
-    async def stop(self) -> None:
-        if self._browser is not None:
-            await self._browser.close()
-            self._browser = None
-        if self._playwright is not None:
-            await self._playwright.stop()
-            self._playwright = None
-
-    @asynccontextmanager
-    async def page(self):
-        if self._browser is None:
-            raise RuntimeError("BrowserManager is not started")
-
-        context = await self._browser.new_context(
-            user_agent=(
-                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-                "AppleWebKit/537.36 (KHTML, like Gecko) "
-                "Chrome/153.0.0.0 Safari/537.36"
-            ),
-            viewport=ViewportSize(**{"width": 1920, "height": 1080}),
-            locale="en-GB",
-            extra_http_headers={
-                "Accept-Language": "en-GB,en;q=0.9,en-US;q=0.8,en;q=0.7",
-            },
-        )
-        page = await context.new_page()
-        try:
-            yield page
-        finally:
-            await context.close()

fetta-0.2.0/src/fetta/protocol.py

@@ -1,8 +0,0 @@
-from typing import Protocol
-
-from fetta.models import FetchResult
-
-
-class FetcherProtocol(Protocol):
-    async def fetch(self, url: str, **kwargs) -> FetchResult: ...
-    async def close(self) -> None: ...