souppot 0.1.0__tar.gz

souppot-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,44 @@
+name: CI
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+jobs:
+  quality:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install Hatch
+        run: python -m pip install --upgrade pip hatch
+
+      - name: Check formatting
+        run: hatch run ruff format --check .
+
+      - name: Lint
+        run: hatch run ruff check .
+
+      - name: Type check
+        run: hatch run mypy src/souppot
+
+      - name: Run unit tests
+        run: hatch run pytest tests/test_core.py
+
+      - name: Install Playwright Chromium
+        run: hatch run python -m playwright install --with-deps chromium
+
+      - name: Run functional tests
+        run: hatch run pytest tests/functional
+
+      - name: Build package
+        run: hatch run python -m build

souppot-0.1.0/.github/workflows/docs.yml

@@ -0,0 +1,51 @@
+name: Docs
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - docs
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: pages
+  cancel-in-progress: false
+
+jobs:
+  deploy:
+    name: Deploy
+    runs-on: ubuntu-latest
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install Hatch
+        run: python -m pip install --upgrade pip hatch
+
+      - name: Build docs
+        run: hatch run docs:build
+
+      - name: Configure Pages
+        uses: actions/configure-pages@v5
+
+      - name: Upload Pages artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: docs/_build/html
+
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

souppot-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,51 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - "v*"
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  id-token: write
+
+jobs:
+  build:
+    name: Build and publish distribution packages
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/souppot
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install build tooling
+        run: python -m pip install --upgrade pip build twine
+
+      - name: Build sdist and wheel
+        run: python -m build
+
+      - name: Check distribution metadata
+        run: twine check dist/*
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          skip-existing: true
+
+      - name: Generate SHA256 checksums
+        run: sha256sum dist/* > dist/checksums.txt
+
+      - name: Upload artifacts to GitHub Release
+        uses: softprops/action-gh-release@v3
+        with:
+          files: dist/*
+          generate_release_notes: true

souppot-0.1.0/.gitignore

@@ -0,0 +1,10 @@
+__pycache__/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+
+build/
+dist/
+*.egg-info/
+
+docs/_build/

souppot-0.1.0/CHANGELOG.md

@@ -0,0 +1,9 @@
+# Changelog
+
+## v0.1.0
+
+Initial release.
+
+- Added `cold_soup` for fetching static HTML with `requests`.
+- Added `hot_soup` for parsing JavaScript-rendered pages with Playwright Chromium.
+- Added `hot_pot` for downloading files through Playwright's request context.

souppot-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 souppot contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

souppot-0.1.0/PKG-INFO

@@ -0,0 +1,25 @@
+Metadata-Version: 2.4
+Name: souppot
+Version: 0.1.0
+Summary: Small helpers for fetching and parsing HTML with requests or Playwright.
+Project-URL: Repository, https://github.com/octanima-labs/souppot
+Project-URL: Documentation, https://octanima-labs.github.io/souppot/
+Project-URL: Issues, https://github.com/octanima-labs/souppot/issues
+Author-email: octanima-labs <octanima@tuta.io>
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.11
+Requires-Dist: 2ning
+Requires-Dist: beautifulsoup4
+Requires-Dist: playwright
+Requires-Dist: requests

souppot-0.1.0/README.md

@@ -0,0 +1,73 @@
+# souppot
+
+Small helpers for fetching and parsing HTML with `requests` or Playwright.
+
+## Installation
+
+From a checkout:
+
+```bash
+pip install .
+```
+
+When published to PyPI:
+
+```bash
+pip install souppot
+```
+
+For JavaScript-rendered pages and Playwright-backed downloads, install Chromium:
+
+```bash
+python -m playwright install chromium
+```
+
+## Usage
+
+Fetch static HTML:
+
+```python
+from souppot import cold_soup
+
+soup = cold_soup("https://example.com")
+
+if soup:
+    print(soup.title.string)
+```
+
+Fetch JavaScript-rendered HTML:
+
+```python
+from souppot import hot_soup
+
+soup = hot_soup("https://example.com", wait_selector=".loaded")
+
+if soup:
+    print(soup.select_one(".loaded").get_text(strip=True))
+```
+
+Download a file with Playwright:
+
+```python
+from souppot import hot_pot
+
+path = hot_pot(
+    "https://example.com/file.zip",
+    "downloads/file.zip",
+    referer="https://example.com",
+)
+
+print(path)
+```
+
+## Documentation
+
+The extended API documentation can be found [here](https://octanima-labs.github.io/souppot/).
+
+## Changelog
+
+See [CHANGELOG.md](CHANGELOG.md).
+
+## License
+
+MIT. See [LICENSE](LICENSE).

souppot-0.1.0/docs/api.rst

@@ -0,0 +1,5 @@
+API Reference
+=============
+
+.. automodule:: souppot
+   :members: cold_soup, hot_soup, hot_pot

souppot-0.1.0/docs/conf.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "src"))
+
+project = "souppot"
+copyright = "2026, souppot contributors"
+author = "souppot contributors"
+release = "0.1.0"
+
+extensions = [
+    "myst_parser",
+    "sphinx.ext.autodoc",
+    "sphinx.ext.intersphinx",
+    "sphinx.ext.napoleon",
+    "sphinx_autodoc_typehints",
+]
+
+source_suffix = {
+    ".md": "markdown",
+    ".rst": "restructuredtext",
+}
+master_doc = "index"
+
+html_theme = "pydata_sphinx_theme"
+html_title = "souppot"
+html_sidebars = {
+    "**": [],
+}
+html_theme_options = {
+    "show_toc_level": 2,
+}
+
+autodoc_default_options = {
+    "members": True,
+    "show-inheritance": True,
+}
+autodoc_typehints = "description"
+autodoc_typehints_format = "short"
+napoleon_google_docstring = True
+napoleon_numpy_docstring = False
+
+intersphinx_mapping = {
+    "python": ("https://docs.python.org/3", None),
+    "requests": ("https://requests.readthedocs.io/en/latest/", None),
+    "bs4": ("https://www.crummy.com/software/BeautifulSoup/bs4/doc/", None),
+}

souppot-0.1.0/docs/index.md

@@ -0,0 +1,37 @@
+# souppot
+
+Small helpers for fetching and parsing HTML with `requests` or Playwright.
+
+Use `cold_soup` for normal server-rendered HTML, `hot_soup` for JavaScript-rendered pages, and `hot_pot` when a download needs Playwright's browser-like request stack.
+
+```{toctree}
+:maxdepth: 2
+:caption: Contents
+
+usage
+api
+```
+
+## Installation
+
+From a checkout:
+
+```bash
+pip install .
+```
+
+When published to PyPI:
+
+```bash
+pip install souppot
+```
+
+For JavaScript-rendered pages and Playwright-backed downloads, install Chromium:
+
+```bash
+python -m playwright install chromium
+```
+
+## License
+
+souppot is released under the MIT license.

souppot-0.1.0/docs/usage.md

@@ -0,0 +1,49 @@
+# Usage
+
+## Static HTML
+
+Use `cold_soup` for pages that do not require JavaScript rendering.
+
+```python
+from souppot import cold_soup
+
+soup = cold_soup("https://example.com")
+
+if soup:
+    print(soup.title.string)
+```
+
+`cold_soup` returns a `BeautifulSoup` object for HTML responses, a raw `requests.Response` for other successful response types, and `None` for missing URLs or non-200 responses.
+
+## JavaScript-Rendered HTML
+
+Use `hot_soup` when a page needs Playwright Chromium to render JavaScript before parsing.
+
+```python
+from souppot import hot_soup
+
+soup = hot_soup("https://example.com", wait_selector=".loaded")
+
+if soup:
+    print(soup.select_one(".loaded").get_text(strip=True))
+```
+
+If `wait_selector` times out, `hot_soup` logs the timeout and parses whatever DOM is available.
+
+## Downloads
+
+Use `hot_pot` when a file should be downloaded through Playwright's request context.
+
+```python
+from souppot import hot_pot
+
+path = hot_pot(
+    "https://example.com/file.zip",
+    "downloads/file.zip",
+    referer="https://example.com",
+)
+
+print(path)
+```
+
+Parent directories for the destination path are created automatically.

souppot-0.1.0/pyproject.toml

@@ -0,0 +1,89 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "souppot"
+version = "0.1.0"
+description = "Small helpers for fetching and parsing HTML with requests or Playwright."
+license = "MIT"
+authors = [
+  { name = "octanima-labs", email = "octanima@tuta.io" },
+]
+requires-python = ">=3.11"
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Intended Audience :: Developers",
+  "License :: OSI Approved :: MIT License",
+  "Operating System :: OS Independent",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Topic :: Internet :: WWW/HTTP",
+  "Topic :: Software Development :: Libraries :: Python Modules",
+]
+dependencies = [
+  "beautifulsoup4",
+  "requests",
+  "playwright",
+  "2ning",
+]
+
+[project.urls]
+Repository = "https://github.com/octanima-labs/souppot"
+Documentation = "https://octanima-labs.github.io/souppot/"
+Issues = "https://github.com/octanima-labs/souppot/issues"
+
+[dependency-groups]
+dev = [
+  "build",
+  "mypy",
+  "pytest",
+  "ruff",
+]
+docs = [
+  "myst-parser",
+  "pydata-sphinx-theme",
+  "sphinx",
+  "sphinx-autodoc-typehints",
+]
+
+[tool.ruff]
+target-version = "py311"
+
+[tool.mypy]
+python_version = "3.11"
+
+[[tool.mypy.overrides]]
+module = ["tuning"]
+ignore_missing_imports = true
+
+[tool.hatch.envs.default]
+dependencies = [
+  "build",
+  "mypy",
+  "pytest",
+  "ruff",
+]
+
+[tool.hatch.envs.docs]
+dependencies = [
+  "myst-parser",
+  "pydata-sphinx-theme",
+  "sphinx",
+  "sphinx-autodoc-typehints",
+]
+
+[tool.hatch.envs.docs.scripts]
+build = "sphinx-build -W -b html docs docs/_build/html"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/souppot"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["src"]
+markers = [
+  "functional: local HTTP server tests that exercise real requests and Playwright paths",
+]

souppot-0.1.0/src/souppot/__init__.py

@@ -0,0 +1,5 @@
+"""Public package interface for souppot HTML fetching helpers."""
+
+from .core import cold_soup, hot_pot, hot_soup
+
+__all__ = ("cold_soup", "hot_soup", "hot_pot")

souppot-0.1.0/src/souppot/core.py

@@ -0,0 +1,261 @@
+"""Core helpers for fetching static pages, rendered pages, and downloads.
+
+``cold_soup`` uses ``requests`` for normal HTTP responses. ``hot_soup`` and
+``hot_pot`` use Playwright Chromium for JavaScript-rendered pages and
+browser-like download requests.
+"""
+
+import time
+from pathlib import Path
+from typing import Final
+from urllib.parse import urlparse
+
+import requests
+import tuning
+from bs4 import BeautifulSoup
+from playwright.sync_api import Error as PlaywrightError
+from playwright.sync_api import Browser
+from playwright.sync_api import BrowserContext
+from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
+from playwright.sync_api import sync_playwright
+
+
+logger = tuning.getLogger(__name__)
+
+BROWSER_USER_AGENT: Final[str] = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+    "AppleWebKit/537.36 (KHTML, like Gecko) "
+    "Chrome/122.0.0.0 Safari/537.36"
+)
+HTML_ACCEPT: Final[str] = (
+    "text/html,application/xhtml+xml,application/xml;q=0.9,"
+    "image/avif,image/webp,image/apng,*/*;q=0.8"
+)
+HTML_HEADERS: Final[dict[str, str]] = {
+    "User-Agent": BROWSER_USER_AGENT,
+    "Accept": HTML_ACCEPT,
+    "Accept-Language": "en-US,en;q=0.9",
+    "Accept-Encoding": "gzip, deflate, br",
+    "Connection": "keep-alive",
+    "Upgrade-Insecure-Requests": "1",
+    "Cache-Control": "no-cache",
+    "Pragma": "no-cache",
+    "DNT": "1",
+    "Sec-Fetch-Dest": "document",
+    "Sec-Fetch-Mode": "navigate",
+    "Sec-Fetch-Site": "none",
+    "Sec-Fetch-User": "?1",
+}
+PLAYWRIGHT_HTML_HEADERS: Final[dict[str, str]] = {
+    "Accept": HTML_ACCEPT,
+    "Accept-Language": "en-US,en;q=0.9",
+    "Cache-Control": "no-cache",
+    "Pragma": "no-cache",
+    "DNT": "1",
+    "Upgrade-Insecure-Requests": "1",
+}
+DOWNLOAD_HEADERS: Final[dict[str, str]] = {
+    "Accept": "application/octet-stream,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.9",
+    "Cache-Control": "no-cache",
+    "Pragma": "no-cache",
+}
+
+__all__: Final[tuple[str, ...]] = ("cold_soup", "hot_soup", "hot_pot")
+
+
+def _clean_url(url: str | None) -> str | None:
+    """Strip URL input and normalize missing values to ``None``."""
+    if url is None:
+        return None
+    url = str(url).strip()
+    return url or None
+
+
+def cold_soup(
+    url: str | None,
+    check_errors: bool = False,
+) -> BeautifulSoup | requests.Response | None:
+    """Fetch a URL with ``requests`` and parse HTML responses.
+
+    Args:
+        url: URL to fetch. ``None`` and blank strings are treated as missing.
+        check_errors: If true, call ``raise_for_status()`` before normal status
+            handling.
+
+    Returns:
+        ``BeautifulSoup`` for ``200`` responses with a ``text/html`` content
+        type, the raw ``requests.Response`` for other ``200`` responses, and
+        ``None`` for missing URLs or non-``200`` responses.
+
+    Raises:
+        requests.HTTPError: If ``check_errors`` is true and the response status
+            is an HTTP error.
+    """
+    url = _clean_url(url)
+    if url is None:
+        logger.warning("URL not provided")
+        return None
+    logger.debug("GET %s", url)
+    parsed = urlparse(url)
+    origin = (
+        f"{parsed.scheme}://{parsed.netloc}"
+        if parsed.scheme and parsed.netloc
+        else None
+    )
+    headers = HTML_HEADERS.copy()
+    if origin:
+        headers["Referer"] = origin + "/"
+
+    res = requests.get(url=url, headers=headers, timeout=15, allow_redirects=True)
+    if check_errors:
+        res.raise_for_status()
+    if res.history:
+        logger.debug("Redirected (%s hops) -> %s", len(res.history), res.url)
+        for hop in res.history:
+            logger.debug("    %s %s", hop.status_code, hop.url)
+    if res.status_code != 200:
+        logger.error("HTTP error: %s", res.status_code)
+        return None
+
+    ct = res.headers.get("Content-Type", "").lower()
+    if "text/html" not in ct:
+        logger.debug("Not an HTML page (%s)", ct)
+        return res
+
+    soup = BeautifulSoup(res.text, "html.parser")
+    logger.debug("✅ Soup is served (%s chars)", len(res.text))
+    return soup
+
+
+def hot_soup(
+    url: str | None,
+    wait_seconds: float = 3,
+    wait_selector: str | None = None,
+) -> BeautifulSoup | None:
+    """Render a URL with Playwright Chromium and parse the final DOM.
+
+    Args:
+        url: URL to render. ``None`` and blank strings are treated as missing.
+        wait_seconds: Seconds to sleep after ``domcontentloaded`` when no
+            ``wait_selector`` is provided. When waiting for a selector, this is
+            converted to the selector timeout with a minimum of 1000 ms.
+        wait_selector: Optional CSS selector to wait for before parsing. If the
+            selector times out, the currently rendered DOM is parsed anyway.
+
+    Returns:
+        ``BeautifulSoup`` for the rendered page, or ``None`` for missing URLs or
+        Playwright errors.
+    """
+    url = _clean_url(url)
+    if url is None:
+        logger.warning("URL not provided")
+        return None
+
+    logger.debug("RENDER %s", url)
+    try:
+        with sync_playwright() as p:
+            browser: Browser | None = None
+            context: BrowserContext | None = None
+            try:
+                browser = p.chromium.launch(headless=True)
+                context = browser.new_context(
+                    user_agent=BROWSER_USER_AGENT,
+                    locale="en-US",
+                    viewport={"width": 1920, "height": 1080},
+                )
+                context.set_extra_http_headers(PLAYWRIGHT_HTML_HEADERS.copy())
+
+                page = context.new_page()
+                response = page.goto(url, wait_until="domcontentloaded", timeout=30_000)
+                if page.url != url:
+                    status = response.status if response is not None else "?"
+                    logger.info("Redirected -> %s (status: %s)", page.url, status)
+
+                if wait_selector:
+                    try:
+                        page.wait_for_selector(
+                            wait_selector,
+                            timeout=max(1000, int(wait_seconds * 1000)),
+                        )
+                    except PlaywrightTimeoutError:
+                        logger.error("Timeout waiting for selector: %s", wait_selector)
+                        # Continue anyway and parse whatever has been rendered so far.
+                else:
+                    time.sleep(max(0, float(wait_seconds)))
+
+                html = page.content()
+            finally:
+                if context is not None:
+                    context.close()
+                if browser is not None:
+                    browser.close()
+
+        soup = BeautifulSoup(html, "html.parser")
+        logger.debug("✅ Soup is served (JS-rendered - %s chars)", len(html))
+        return soup
+
+    except PlaywrightError as e:
+        logger.error("Playwright error: %s", e, exc_info=True)
+        return None
+
+
+def hot_pot(
+    url: str | None,
+    dest: str | Path,
+    referer: str | None = None,
+    timeout_ms: int = 60_000,
+) -> Path:
+    """Download a URL with Playwright's request context and save it to disk.
+
+    Args:
+        url: URL to download. ``None`` and blank strings raise ``ValueError``.
+        dest: Destination file path. Parent directories are created if needed.
+        referer: Optional ``Referer`` header to send with the request.
+        timeout_ms: Playwright request timeout in milliseconds.
+
+    Returns:
+        The destination path as a ``Path``.
+
+    Raises:
+        ValueError: If ``url`` is missing.
+        playwright.sync_api.Error: If Playwright cannot complete the request.
+    """
+    url = _clean_url(url)
+    if url is None:
+        raise ValueError("URL not provided")
+
+    dest = Path(dest)
+    dest.parent.mkdir(parents=True, exist_ok=True)
+
+    logger.debug("PLAYWRIGHT GET %s", url)
+    with sync_playwright() as p:
+        browser: Browser | None = None
+        context: BrowserContext | None = None
+        try:
+            browser = p.chromium.launch(headless=True)
+            context = browser.new_context(
+                user_agent=BROWSER_USER_AGENT,
+                locale="en-US",
+            )
+
+            headers = DOWNLOAD_HEADERS.copy()
+            if referer:
+                headers["Referer"] = referer
+
+            response = context.request.get(
+                url,
+                headers=headers,
+                fail_on_status_code=True,
+                timeout=timeout_ms,
+            )
+            body = response.body()
+        finally:
+            if context is not None:
+                context.close()
+            if browser is not None:
+                browser.close()
+
+    dest.write_bytes(body)
+    logger.debug("Download saved to: %s", dest)
+    return dest

souppot-0.1.0/tests/fixtures/basic.html

@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <title>Soup Pot Fixture</title>
+  </head>
+  <body>
+    <main id="content">
+      <h1 id="title">Soup Pot</h1>
+      <p class="message">Fixture HTML for parser unit tests.</p>
+    </main>
+  </body>
+</html>

souppot-0.1.0/tests/functional/fixtures/dummy.bin

@@ -0,0 +1 @@
+souppot functional download fixture

souppot-0.1.0/tests/functional/fixtures/page.html

@@ -0,0 +1,21 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <title>Souppot Functional Fixture</title>
+  </head>
+  <body>
+    <main id="content">
+      <h1 id="title">Souppot Functional Fixture</h1>
+      <p class="static">This element is present in the original HTML.</p>
+    </main>
+    <script>
+      setTimeout(() => {
+        const node = document.createElement("p");
+        node.className = "delayed";
+        node.textContent = "This element was created by JavaScript.";
+        document.querySelector("#content").appendChild(node);
+      }, 200);
+    </script>
+  </body>
+</html>

souppot-0.1.0/tests/functional/test_functional.py

@@ -0,0 +1,91 @@
+from functools import partial
+from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+from threading import Thread
+
+import pytest
+from bs4 import BeautifulSoup
+from playwright.sync_api import Error as PlaywrightError
+from playwright.sync_api import sync_playwright
+from souppot import cold_soup, hot_pot, hot_soup
+
+
+pytestmark = pytest.mark.functional
+
+FIXTURES = Path(__file__).parent / "fixtures"
+
+
+class QuietHandler(SimpleHTTPRequestHandler):
+    def log_message(self, format: str, *args: object) -> None:
+        return None
+
+
+@pytest.fixture(scope="module")
+def fixture_server() -> str:
+    handler = partial(QuietHandler, directory=str(FIXTURES))
+    server = ThreadingHTTPServer(("127.0.0.1", 0), handler)
+    thread = Thread(target=server.serve_forever, daemon=True)
+    thread.start()
+
+    try:
+        host, port = server.server_address
+        yield f"http://{host}:{port}"
+    finally:
+        server.shutdown()
+        server.server_close()
+        thread.join(timeout=5)
+
+
+@pytest.fixture(scope="module")
+def chromium_available() -> None:
+    try:
+        with sync_playwright() as p:
+            browser = p.chromium.launch(headless=True)
+            browser.close()
+    except PlaywrightError as exc:
+        pytest.skip(f"Playwright Chromium is not available: {exc}")
+
+
+def test_cold_soup_fetches_local_html(fixture_server: str) -> None:
+    soup = cold_soup(f"{fixture_server}/page.html")
+
+    assert isinstance(soup, BeautifulSoup)
+    assert (
+        soup.select_one("#title").get_text(strip=True) == "Souppot Functional Fixture"
+    )
+    assert (
+        soup.select_one(".static").get_text(strip=True)
+        == "This element is present in the original HTML."
+    )
+    assert soup.select_one(".delayed") is None
+
+
+def test_hot_soup_waits_for_javascript_created_element(
+    fixture_server: str,
+    chromium_available: None,
+) -> None:
+    soup = hot_soup(
+        f"{fixture_server}/page.html", wait_selector=".delayed", wait_seconds=2
+    )
+
+    assert isinstance(soup, BeautifulSoup)
+    assert (
+        soup.select_one(".delayed").get_text(strip=True)
+        == "This element was created by JavaScript."
+    )
+
+
+def test_hot_pot_downloads_local_file(
+    fixture_server: str,
+    chromium_available: None,
+    tmp_path: Path,
+) -> None:
+    source = FIXTURES / "dummy.bin"
+    dest = tmp_path / "downloads" / "dummy.bin"
+
+    result = hot_pot(
+        f"{fixture_server}/dummy.bin", dest, referer=f"{fixture_server}/page.html"
+    )
+
+    assert result == dest
+    assert dest.read_bytes() == source.read_bytes()

souppot-0.1.0/tests/test_core.py

@@ -0,0 +1,353 @@
+from pathlib import Path
+
+import pytest
+import souppot
+from bs4 import BeautifulSoup
+from souppot import core
+
+
+FIXTURES = Path(__file__).parent / "fixtures"
+
+
+class FakeResponse:
+    def __init__(
+        self,
+        *,
+        status_code: int = 200,
+        text: str = "",
+        headers: dict[str, str] | None = None,
+        url: str = "https://example.com/page",
+        history: list[object] | None = None,
+        error: Exception | None = None,
+    ) -> None:
+        self.status_code = status_code
+        self.text = text
+        self.headers = headers or {}
+        self.url = url
+        self.history = history or []
+        self.error = error
+
+    def raise_for_status(self) -> None:
+        if self.error is not None:
+            raise self.error
+
+
+class FakeRenderedResponse:
+    status = 200
+
+
+class FakePage:
+    def __init__(self, html: str, *, wait_raises: bool = False) -> None:
+        self.html = html
+        self.url = "https://example.com/page"
+        self.wait_raises = wait_raises
+        self.wait_selector_calls: list[tuple[str, int]] = []
+
+    def goto(self, url: str, *, wait_until: str, timeout: int) -> FakeRenderedResponse:
+        self.url = url
+        self.goto_call = {"url": url, "wait_until": wait_until, "timeout": timeout}
+        return FakeRenderedResponse()
+
+    def wait_for_selector(self, selector: str, *, timeout: int) -> None:
+        self.wait_selector_calls.append((selector, timeout))
+        if self.wait_raises:
+            raise core.PlaywrightTimeoutError("selector timed out")
+
+    def content(self) -> str:
+        return self.html
+
+
+class FakeBrowserContext:
+    def __init__(self, page: FakePage) -> None:
+        self.page = page
+        self.extra_headers: dict[str, str] | None = None
+        self.closed = False
+
+    def set_extra_http_headers(self, headers: dict[str, str]) -> None:
+        self.extra_headers = headers
+
+    def new_page(self) -> FakePage:
+        return self.page
+
+    def close(self) -> None:
+        self.closed = True
+
+
+class FakeBrowser:
+    def __init__(self, context: FakeBrowserContext) -> None:
+        self.context = context
+        self.closed = False
+
+    def new_context(self, **kwargs: object) -> FakeBrowserContext:
+        self.new_context_kwargs = kwargs
+        return self.context
+
+    def close(self) -> None:
+        self.closed = True
+
+
+class FakeChromium:
+    def __init__(self, browser: FakeBrowser) -> None:
+        self.browser = browser
+
+    def launch(self, *, headless: bool) -> FakeBrowser:
+        self.launch_kwargs = {"headless": headless}
+        return self.browser
+
+
+class FakeSyncPlaywright:
+    def __init__(self, chromium: FakeChromium) -> None:
+        self.chromium = chromium
+
+    def __enter__(self) -> "FakeSyncPlaywright":
+        return self
+
+    def __exit__(self, *args: object) -> None:
+        return None
+
+
+class FakeDownloadResponse:
+    def __init__(self, body: bytes) -> None:
+        self._body = body
+
+    def body(self) -> bytes:
+        return self._body
+
+
+class FakeRequestContext:
+    def __init__(self, body: bytes) -> None:
+        self.body = body
+        self.calls: list[dict[str, object]] = []
+
+    def get(self, url: str, **kwargs: object) -> FakeDownloadResponse:
+        self.calls.append({"url": url, **kwargs})
+        return FakeDownloadResponse(self.body)
+
+
+class FakeDownloadContext:
+    def __init__(self, body: bytes) -> None:
+        self.request = FakeRequestContext(body)
+        self.closed = False
+
+    def close(self) -> None:
+        self.closed = True
+
+
+class FakeDownloadBrowser:
+    def __init__(self, context: FakeDownloadContext) -> None:
+        self.context = context
+        self.closed = False
+
+    def new_context(self, **kwargs: object) -> FakeDownloadContext:
+        self.new_context_kwargs = kwargs
+        return self.context
+
+    def close(self) -> None:
+        self.closed = True
+
+
+class FakeDownloadChromium:
+    def __init__(self, browser: FakeDownloadBrowser) -> None:
+        self.browser = browser
+
+    def launch(self, *, headless: bool) -> FakeDownloadBrowser:
+        self.launch_kwargs = {"headless": headless}
+        return self.browser
+
+
+@pytest.fixture
+def fixture_html() -> str:
+    return (FIXTURES / "basic.html").read_text(encoding="utf-8")
+
+
+def test_package_exports_public_api() -> None:
+    assert souppot.__all__ == ("cold_soup", "hot_soup", "hot_pot")
+    assert souppot.cold_soup is core.cold_soup
+    assert souppot.hot_soup is core.hot_soup
+    assert souppot.hot_pot is core.hot_pot
+
+
+@pytest.mark.parametrize("url", [None, "", "   "])
+def test_cold_soup_missing_url_returns_none_without_request(
+    monkeypatch: pytest.MonkeyPatch, url: str | None
+) -> None:
+    def fail_get(**kwargs: object) -> None:
+        raise AssertionError("requests.get should not be called")
+
+    monkeypatch.setattr(core.requests, "get", fail_get)
+
+    assert core.cold_soup(url) is None
+
+
+def test_cold_soup_sends_browser_like_request_headers(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    response = FakeResponse(headers={"Content-Type": "application/json"})
+    calls: list[dict[str, object]] = []
+
+    def fake_get(**kwargs: object) -> FakeResponse:
+        calls.append(kwargs)
+        return response
+
+    monkeypatch.setattr(core.requests, "get", fake_get)
+
+    assert core.cold_soup(" https://example.com/path ") is response
+    call = calls[0]
+    headers = call["headers"]
+
+    assert call["url"] == "https://example.com/path"
+    assert "stream" not in call
+    assert call["timeout"] == 15
+    assert call["allow_redirects"] is True
+    assert isinstance(headers, dict)
+    assert "Mozilla/5.0" in headers["User-Agent"]
+    assert headers["Referer"] == "https://example.com/"
+
+
+def test_cold_soup_returns_beautifulsoup_for_html_response(
+    monkeypatch: pytest.MonkeyPatch,
+    fixture_html: str,
+) -> None:
+    response = FakeResponse(
+        text=fixture_html, headers={"Content-Type": "text/html; charset=utf-8"}
+    )
+
+    monkeypatch.setattr(core.requests, "get", lambda **kwargs: response)
+
+    soup = core.cold_soup("https://example.com/page")
+
+    assert isinstance(soup, BeautifulSoup)
+    assert soup.select_one("#title").get_text(strip=True) == "Soup Pot"
+
+
+def test_cold_soup_returns_response_for_non_html_response(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    response = FakeResponse(
+        text='{"ok": true}', headers={"Content-Type": "application/json"}
+    )
+
+    monkeypatch.setattr(core.requests, "get", lambda **kwargs: response)
+
+    assert core.cold_soup("https://example.com/data.json") is response
+
+
+def test_cold_soup_returns_none_for_non_200_response(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    response = FakeResponse(status_code=404, headers={"Content-Type": "text/html"})
+
+    monkeypatch.setattr(core.requests, "get", lambda **kwargs: response)
+
+    assert core.cold_soup("https://example.com/missing") is None
+
+
+def test_cold_soup_check_errors_raises_before_status_handling(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    class MarkerError(Exception):
+        pass
+
+    response = FakeResponse(status_code=500, error=MarkerError("boom"))
+
+    monkeypatch.setattr(core.requests, "get", lambda **kwargs: response)
+
+    with pytest.raises(MarkerError):
+        core.cold_soup("https://example.com/error", check_errors=True)
+
+
+@pytest.mark.parametrize("url", [None, "", "   "])
+def test_hot_soup_missing_url_returns_none_without_playwright(
+    monkeypatch: pytest.MonkeyPatch, url: str | None
+) -> None:
+    def fail_sync_playwright() -> None:
+        raise AssertionError("sync_playwright should not be called")
+
+    monkeypatch.setattr(core, "sync_playwright", fail_sync_playwright)
+
+    assert core.hot_soup(url) is None
+
+
+def test_hot_soup_parses_rendered_html_from_fake_playwright(
+    monkeypatch: pytest.MonkeyPatch,
+    fixture_html: str,
+) -> None:
+    page = FakePage(fixture_html)
+    context = FakeBrowserContext(page)
+    browser = FakeBrowser(context)
+    playwright = FakeSyncPlaywright(FakeChromium(browser))
+    sleep_calls: list[float] = []
+
+    monkeypatch.setattr(core, "sync_playwright", lambda: playwright)
+    monkeypatch.setattr(core.time, "sleep", lambda seconds: sleep_calls.append(seconds))
+
+    soup = core.hot_soup("https://example.com/page", wait_seconds=0)
+
+    assert isinstance(soup, BeautifulSoup)
+    assert (
+        soup.select_one(".message").get_text(strip=True)
+        == "Fixture HTML for parser unit tests."
+    )
+    assert sleep_calls == [0]
+    assert context.closed is True
+    assert browser.closed is True
+
+
+def test_hot_soup_wait_selector_timeout_still_parses_html(
+    monkeypatch: pytest.MonkeyPatch,
+    fixture_html: str,
+) -> None:
+    page = FakePage(fixture_html, wait_raises=True)
+    context = FakeBrowserContext(page)
+    browser = FakeBrowser(context)
+    playwright = FakeSyncPlaywright(FakeChromium(browser))
+
+    monkeypatch.setattr(core, "sync_playwright", lambda: playwright)
+
+    soup = core.hot_soup(
+        "https://example.com/page", wait_selector="#missing", wait_seconds=0.2
+    )
+
+    assert isinstance(soup, BeautifulSoup)
+    assert soup.select_one("#content") is not None
+    assert page.wait_selector_calls == [("#missing", 1000)]
+
+
+@pytest.mark.parametrize("url", [None, "", "   "])
+def test_hot_pot_missing_url_raises_value_error(
+    url: str | None, tmp_path: Path
+) -> None:
+    with pytest.raises(ValueError, match="URL not provided"):
+        core.hot_pot(url, tmp_path / "out.bin")
+
+
+def test_hot_pot_creates_parent_dirs_and_writes_body(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+    body = b"downloaded bytes"
+    context = FakeDownloadContext(body)
+    browser = FakeDownloadBrowser(context)
+    playwright = FakeSyncPlaywright(FakeDownloadChromium(browser))
+    dest = tmp_path / "nested" / "out.bin"
+
+    monkeypatch.setattr(core, "sync_playwright", lambda: playwright)
+
+    result = core.hot_pot(
+        " https://example.com/file.bin ",
+        dest,
+        referer="https://example.com/page",
+        timeout_ms=123,
+    )
+
+    assert result == dest
+    assert dest.read_bytes() == body
+    assert context.closed is True
+    assert browser.closed is True
+
+    call = context.request.calls[0]
+    headers = call["headers"]
+    assert call["url"] == "https://example.com/file.bin"
+    assert call["fail_on_status_code"] is True
+    assert call["timeout"] == 123
+    assert isinstance(headers, dict)
+    assert headers["Referer"] == "https://example.com/page"