fetta 0.1.0__py3-none-any.whl

fetta/__init__.py

@@ -0,0 +1,8 @@
+from fetta.browser import BrowserManager
+from fetta.fetchers import BrowserFetcher, HttpFetcher
+from fetta.json_fetcher import JsonFetcher
+from fetta.models import PageContent
+from fetta.protocol import FetcherProtocol
+from fetta.requests_fetcher import RequestsFetcher
+
+__all__ = ["BrowserManager", "PageContent", "BrowserFetcher", "HttpFetcher", "FetcherProtocol"]

fetta/base.py

@@ -0,0 +1,61 @@
+from abc import ABC, abstractmethod
+from typing import Any
+
+from bs4 import BeautifulSoup
+
+from fetta.models import PageContent
+
+# Tags that never contain useful article text.
+# Kept module-level so subclasses can reference the same tuple.
+_STRIP_TAGS = ("script", "style", "nav", "footer", "header", "noscript")
+
+
+class BaseFetcher(ABC):
+    """
+    Abstract base class for fetchers.
+    """
+
+    @abstractmethod
+    async def fetch(self, url: str, **kwargs: Any) -> PageContent:
+        """Fetch a page and return its parsed content."""
+        pass
+
+    @abstractmethod
+    async def close(self) -> None:
+        """Release resources (HTTP connections, browser context, etc.)."""
+        pass
+
+    async def __aenter__(self) -> "BaseFetcher":
+        return self
+
+    async def __aexit__(self, exc_type, exc, tb) -> None:
+        await self.close()
+
+    def _parse(
+        self, html: str, url: str, status: int, title: str = "", data: list[dict] = None
+    ) -> PageContent:
+        """
+        Turn raw HTML into a PageContent.
+
+        Extracts the title (if not already provided), strips boilerplate
+        tags, and normalizes the remaining text.
+
+        Args:
+            html: Raw HTML as returned by the transport.
+            url: Original URL (stored on the result for traceability).
+            status: HTTP status code.
+            title: Pre-extracted title. If empty, the <title> tag is used.
+        """
+        soup = BeautifulSoup(html, "html.parser")
+
+        # Extract title before stripping tags as some sites put <style>
+        # inside <head>, and decomposing early keeps the tree clean.
+        if not title and soup.title is not None:
+            title = soup.title.get_text(strip=True)
+
+        for tag in soup(_STRIP_TAGS):
+            tag.decompose()
+
+        text = soup.get_text(separator="\n", strip=True)
+
+        return PageContent(url=url, title=title, text=text, html=html, data=data, status=status)

fetta/browser.py

@@ -0,0 +1,49 @@
+from contextlib import asynccontextmanager
+
+from playwright.async_api import (
+    Browser,
+    Playwright,
+    async_playwright,
+    ViewportSize,
+)
+
+
+class BrowserManager:
+    def __init__(self) -> None:
+        self._playwright: Playwright | None = None
+        self._browser: Browser | None = None
+
+    async def start(self) -> None:
+        self._playwright = await async_playwright().start()
+        self._browser = await self._playwright.chromium.launch(headless=True)
+
+    async def stop(self) -> None:
+        if self._browser is not None:
+            await self._browser.close()
+            self._browser = None
+        if self._playwright is not None:
+            await self._playwright.stop()
+            self._playwright = None
+
+    @asynccontextmanager
+    async def page(self):
+        if self._browser is None:
+            raise RuntimeError("BrowserManager is not started")
+
+        context = await self._browser.new_context(
+            user_agent=(
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                "AppleWebKit/537.36 (KHTML, like Gecko) "
+                "Chrome/153.0.0.0 Safari/537.36"
+            ),
+            viewport=ViewportSize(**{"width": 1920, "height": 1080}),
+            locale="en-GB",
+            extra_http_headers={
+                "Accept-Language": "en-GB,en;q=0.9,en-US;q=0.8,en;q=0.7",
+            },
+        )
+        page = await context.new_page()
+        try:
+            yield page
+        finally:
+            await context.close()

fetta/fetchers.py

@@ -0,0 +1,53 @@
+from typing import Any
+
+from fetta.protocol import FetcherProtocol
+from fetta.base import BaseFetcher
+from fetta.browser import BrowserManager
+from fetta.models import PageContent
+from fetta.playwright_fetcher import PlaywrightFetcher
+from fetta.requests_fetcher import RequestsFetcher
+
+
+class HttpFetcher(BaseFetcher):
+    """
+    Fetches pages over plain HTTP (no browser, no JavaScript).
+
+    A facade over pluggable HTTP engines. By default, it uses
+    RequestsFetcher (sync `requests` offloaded to a worker thread).
+    """
+
+    def __init__(
+        self,
+        *,
+        engine: FetcherProtocol | None = None,
+    ) -> None:
+        self._engine: FetcherProtocol = engine or RequestsFetcher()
+
+    async def fetch(self, url: str, **kwargs: Any) -> PageContent:
+        return await self._engine.fetch(url, **kwargs)
+
+    async def close(self) -> None:
+        await self._engine.close()
+
+
+class BrowserFetcher(BaseFetcher):
+    """
+    Fetches pages through a real browser.
+
+    The engine is an implementation detail. Currently, Playwright is
+    used; alternative engines can be injected via the `engine` argument.
+    """
+
+    def __init__(
+        self,
+        manager: BrowserManager,
+        *,
+        engine: FetcherProtocol | None = None,
+    ) -> None:
+        self._engine: FetcherProtocol = engine or PlaywrightFetcher(manager)
+
+    async def fetch(self, url: str, **kwargs: Any) -> PageContent:
+        return await self._engine.fetch(url, **kwargs)
+
+    async def close(self) -> None:
+        await self._engine.close()

fetta/json_fetcher.py

@@ -0,0 +1,65 @@
+from fetta.base import BaseFetcher
+from fetta.browser import BrowserManager
+from fetta.models import PageContent
+
+
+_BLOCK = {"image", "media", "font"}
+
+
+class JsonFetcher(BaseFetcher):
+    def __init__(self, browser: BrowserManager) -> None:
+        self._browser = browser
+
+    async def fetch(
+        self,
+        url: str,
+        *,
+        wait_until: str = "load",
+        timeout_ms: int = 30_000,
+        **kwargs,
+    ) -> PageContent:
+
+        json_responses: list[dict] = []
+
+        async def on_response(response):
+            content_type = response.headers.get("content-type", "")
+            if "application/json" in content_type:
+                try:
+                    data = await response.json()
+                    json_responses.append(
+                        {
+                            "url": response.url,
+                            "status": response.status,
+                            "method": response.request.method,
+                            "data": data,
+                        }
+                    )
+                except Exception:
+                    pass  # not valid JSON, skip
+
+        async with self._browser.page() as page:
+            page.on("response", on_response)
+
+            await page.route("**/*", self._block_resources)
+
+            response = await page.goto(url, wait_until=wait_until, timeout=timeout_ms)
+
+            # Give XHR calls that start after load time to complete
+            await page.wait_for_timeout(500)
+
+            html = await page.content()
+            title = await page.title()
+
+        return self._parse(html, url, response.status if response else 0, title, json_responses)
+
+    async def close(self) -> None:
+        # Browser lifecycle is owned by BrowserManager, not by the fetcher.
+        # Nothing to release here, but the method exists to satisfy the contract.
+        pass
+
+    async def _block_resources(self, route):
+        """Abort requests for heavy resources that don't affect text content."""
+        if route.request.resource_type in _BLOCK:
+            await route.abort()
+        else:
+            await route.continue_()

fetta/models.py

@@ -0,0 +1,11 @@
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True, slots=True)
+class PageContent:
+    url: str
+    title: str
+    text: str
+    html: str
+    data: list[dict] | None
+    status: int

fetta/playwright_fetcher.py

@@ -0,0 +1,56 @@
+from fetta.base import BaseFetcher
+from fetta.browser import BrowserManager
+from fetta.models import PageContent
+
+
+_BLOCK = {"image", "media", "font"}
+
+
+class PlaywrightFetcher(BaseFetcher):
+    def __init__(self, browser: BrowserManager) -> None:
+        self._browser = browser
+
+    async def fetch(
+        self,
+        url: str,
+        *,
+        wait_for: str | None = None,
+        wait_until: str = "load",
+        timeout_ms: int = 30_000,
+        **kwargs,
+    ) -> PageContent:
+        async with self._browser.page() as page:
+            await page.route("**/*", self._block_resources)
+
+            response = await page.goto(url=url, wait_until=wait_until, timeout=timeout_ms)
+            if wait_for:
+                await page.wait_for_selector(wait_for, timeout=timeout_ms)
+            else:
+                # If unknown page, wait for body to fill in, cap at 5s
+                # try:
+                #     await page.wait_for_function(
+                #         "() => document.body && document.body.innerText.trim().length > 200",
+                #         timeout=5_000,
+                #     )
+                # except Exception:
+                #     pass
+
+                # Small extra settle for late JS
+                await page.wait_for_timeout(500)
+
+            html = await page.content()
+            title = await page.title()
+
+        return self._parse(html, url, response.status if response else 0, title)
+
+    async def close(self) -> None:
+        # Browser lifecycle is owned by BrowserManager, not by the fetcher.
+        # Nothing to release here, but the method exists to satisfy the contract.
+        pass
+
+    async def _block_resources(self, route):
+        """Abort requests for heavy resources that don't affect text content."""
+        if route.request.resource_type in _BLOCK:
+            await route.abort()
+        else:
+            await route.continue_()

fetta/protocol.py

@@ -0,0 +1,8 @@
+from typing import Protocol
+
+from fetta.models import PageContent
+
+
+class FetcherProtocol(Protocol):
+    async def fetch(self, url: str, **kwargs) -> PageContent: ...
+    async def close(self) -> None: ...

fetta/requests_fetcher.py

@@ -0,0 +1,53 @@
+import asyncio
+from typing import Any
+
+import requests
+
+from fetta.base import BaseFetcher
+from fetta.models import PageContent
+
+
+class RequestsFetcher(BaseFetcher):
+    """
+    Synchronous requests, offloaded to worker threads.
+    """
+
+    def __init__(self, *, timeout: float = 10.0) -> None:
+        self._session = requests.Session()
+        self._session.headers.update(
+            {
+                "User-Agent": (
+                    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                    "AppleWebKit/537.36 (KHTML, like Gecko) "
+                    "Chrome/149.0.0.0 Safari/537.36"
+                ),
+                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+                "Accept-Language": "en-GB,en;q=0.9,en-US;q=0.8,en;q=0.7",
+                "Accept-Encoding": "gzip, deflate, br",
+                "Connection": "keep-alive",
+                "Upgrade-Insecure-Requests": "1",
+                "Sec-Fetch-Dest": "document",
+                "Sec-Fetch-Mode": "navigate",
+                "Sec-Fetch-Site": "none",
+                "Sec-Fetch-User": "?1",
+                "Cache-Control": "max-age=0",
+                "sec-ch-ua": '"Chromium";v="149", "Not)A;Brand";v="24"',
+                "sec-ch-ua-mobile": "?0",
+                "sec-ch-ua-platform": '"Windows"',
+            }
+        )
+        self._timeout = timeout
+
+    def _get_sync(self, url: str) -> tuple[str, int]:
+        response = self._session.get(url, timeout=self._timeout)
+        response.raise_for_status()
+        return response.text, response.status_code
+
+    async def fetch(self, url: str, **kwargs: Any) -> PageContent:
+        # Hand the blocking call to a thread. The event loop is free
+        # to handle other requests while this thread waits on the socket.
+        html, status = await asyncio.to_thread(self._get_sync, url)
+        return self._parse(html, url, status)
+
+    async def close(self) -> None:
+        self._session.close()

fetta-0.1.0.dist-info/METADATA

@@ -0,0 +1,157 @@
+Metadata-Version: 2.4
+Name: fetta
+Version: 0.1.0
+Summary: Tool that fetches web pages for assistants
+Author: Sergei
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: beautifulsoup4>=4.15.0,<5
+Requires-Dist: playwright==1.63.0
+Requires-Dist: requests>=2.34.2,<3
+Requires-Python: >=3.12
+Project-URL: Homepage, https://github.com/spapulin/fetta
+Project-URL: Repository, https://github.com/spapulin/fetta
+Project-URL: Issues, https://github.com/spapulin/fetta/issues
+Description-Content-Type: text/markdown
+
+# fetta
+
+Tool that fetches web pages for assistants. Give it a URL, get back 
+clean text. Handles static pages and SPAs
+
+## ⚡ Async-first
+
+**fetta is async-only.** Every fetcher is an `async def`, every
+method is awaited. There is no sync API.
+
+This is deliberate. `fetta` is built for the runtimes where agents
+actually run MCP servers, LangChain/LangGraph pipelines, FastAPI
+backends. All of these are async. A sync fetcher would block the 
+event loop and stall every other request in the process.
+
+If you're in a sync context, bridge it at your entry point:
+
+```python
+import asyncio
+from fetta import HttpFetcher
+
+async def fetch_content(url: str) -> str:
+    async with HttpFetcher() as fetcher:
+        page = await fetcher.fetch(url)
+        return page.text
+
+# Sync entry point is top of a script, not inside an async function
+text = asyncio.run(fetch_content("https://example.com"))
+```
+
+Don't call `asyncio.run()` from inside an async function that
+raises `RuntimeError`. The bridge is for the outermost layer of
+sync programs only.
+
+## Install
+
+`fetta` is not on PyPI yet. Install directly from GitHub.
+
+### As a library
+
+With uv (recommended):
+
+    uv add "fetta @ git+https://github.com/spapulin/fetta.git"
+
+With pip:
+
+    pip install "fetta @ git+https://github.com/spapulin/fetta.git"
+
+For browser-based fetching, also install Chromium:
+
+    # With uv
+    uv run playwright install chromium --with-deps
+
+    # With pip (venv activated)
+    python -m playwright install chromium --with-deps
+
+`--with-deps` installs system libraries and requires `sudo` on Linux.
+On macOS, drop the flag. 
+
+Skip this step entirely if you only use `HttpFetcher`.
+
+### Set up for development
+
+#### With Docker
+
+    git clone https://github.com/spapulin/fetta.git
+    cd fetta
+
+    make up      # build image, start container
+    make sync    # install dependencies in the container
+    make test    # run tests in the container
+
+Playwright browsers and system libraries are baked into the image.
+
+**Optional**: IDE autocompletion
+
+PyCharm and other IDEs need a local Python interpreter for
+autocompletion and "go to definition". Create a lightweight
+host venv - runtime packages only, no dev tools, no browsers:
+
+    # With uv
+    uv venv
+    uv pip install -e .
+
+    # With pip
+    python -m venv .venv
+    source .venv/bin/activate
+    pip install -e .
+
+Point your IDE at `.venv/bin/python`.
+
+Code still runs inside Docker (`make test`). The host venv is
+only for the editor - it has no Playwright browsers and cannot
+execute `BrowserFetcher`.
+
+#### Without Docker
+
+    git clone https://github.com/spapulin/fetta.git
+    cd fetta
+
+    uv sync --dev
+    uv run playwright install chromium --with-deps
+    uv run pytest
+
+## Quickstart
+
+The fastest way to try `fetta` is the Jupyter notebook:
+
+    notebooks/quickstart.ipynb
+
+It walks through:
+
+- Fetching a static page with `HttpFetcher`
+- Rendering a JavaScript-heavy page with `BrowserFetcher`
+- Using `SmartFetcher` to pick the right engine automatically
+- Extracting text and inspecting the result
+
+### Running the notebook
+
+With Docker (recommended):
+
+    make jupyter
+
+Then open http://localhost:8888 and navigate to `notebooks/quickstart.ipynb`.
+
+Without Docker:
+
+    uv run jupyter lab
+
+## Requirements
+
+- Python 3.12+
+- Playwright browsers (Chromium) - only for `BrowserFetcher` and `SmartFetcher`
+- Docker - only for development
+
+## License
+
+MIT — see [LICENSE](LICENSE)

fetta-0.1.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+fetta/__init__.py,sha256=gtV61mrOZKkoBckhZ9Fi3oV9kEHsxmUAUxJX4BCALx8,367
+fetta/base.py,sha256=n5C_lqiLq6aSu92Z3_yciP_4gpGibCPY8T8wnMs2v3Q,1941
+fetta/browser.py,sha256=jk2eF0HyZG5L2pd6_1-fCdIgPHUgP4d31dOhEbSnp1o,1508
+fetta/fetchers.py,sha256=NIZDLqmiFULPnmJzhIYp13u6dWitjHMU34TBmTJlsQc,1546
+fetta/json_fetcher.py,sha256=XMxDEg3k_GP44ytOd0wRd2SL3EMPY4uOKbxGWJqHvmI,2130
+fetta/models.py,sha256=sy5psqcqQLB4eqqLbWf46rzgPcpFlBYKG3YwTligu2w,191
+fetta/playwright_fetcher.py,sha256=uuNzSt-Sn4iUfp2f9VXTh7LK-XcFlX6DdDxHjqHpkZg,1906
+fetta/protocol.py,sha256=-CdOI6-3PlFssIdfeZ0olC7pNBKV0ilSnvZZKlo4i5M,206
+fetta/requests_fetcher.py,sha256=3gXSLDGkQi0kBkVpPshTIeAqdjANKqnFEz0lsB9kz94,2001
+fetta-0.1.0.dist-info/licenses/LICENSE,sha256=aSpP3-ohcs8WhhfueYi73lOO9DzO6QzUylOULxmei30,1070
+fetta-0.1.0.dist-info/WHEEL,sha256=R1d3uUTbmXM1FHXH_itQashbrqrOSVj-hvBCpmkIIGE,81
+fetta-0.1.0.dist-info/entry_points.txt,sha256=ZF7bQQOhcxB_L1EXj7ra5mTQoZPvRNvPwIGlm9HeQZw,38
+fetta-0.1.0.dist-info/METADATA,sha256=scLqTxoTyEyOeBDFqMiNB--4rCyd3WK2pIMNoIyv84w,4083
+fetta-0.1.0.dist-info/RECORD,,

fetta-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: uv 0.12.17
+Root-Is-Purelib: true
+Tag: py3-none-any

fetta-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+fetta = fetta:main
+

fetta-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Sergei Papulin
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.