pagespring 0.1.0__tar.gz

pagespring-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Mike Farr
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pagespring-0.1.0/PKG-INFO

@@ -0,0 +1,92 @@
+Metadata-Version: 2.4
+Name: pagespring
+Version: 0.1.0
+Summary: Acquire and normalize online software manuals into clean, convertible source files — the acquisition front-end to pagespeak.
+Author: Mike Farr
+License-Expression: MIT
+Project-URL: Repository, https://github.com/phierceweb/pagespring
+Project-URL: Changelog, https://github.com/phierceweb/pagespring/blob/main/CHANGELOG.md
+Project-URL: Issues, https://github.com/phierceweb/pagespring/issues
+Keywords: manuals,documentation,ingest,acquisition,rag,markdown
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Text Processing :: Markup :: Markdown
+Classifier: Typing :: Typed
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: beautifulsoup4>=4.12
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: pf-core[cli]~=0.4.1
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: ruff>=0.6; extra == "dev"
+Requires-Dist: mypy>=1.10; extra == "dev"
+Requires-Dist: pre-commit>=3.7; extra == "dev"
+Dynamic: license-file
+
+# pagespring
+
+Acquire and normalize online software manuals into clean, convertible source
+files — the **acquisition front-end to
+[pagespeak](https://github.com/phierceweb/pagespeak)**.
+
+Point it at a manual's URL. A *pattern* recognizes the source type, *acquires*
+the raw pages (stdlib `urllib`), and *normalizes* them into ONE clean
+HTML/markdown file with absolute asset URLs under `incoming/<slug>/`. That clean
+file is the deliverable; converting it into the finished RAG corpus is a separate
+step (pagespeak) that consumes `incoming/` on its own — pagespring never runs it.
+
+Lean by design: `pf-core[cli]` + `beautifulsoup4`, stdlib fetch, no ML stack.
+
+## Intended use
+
+pagespring is for **publicly available documentation** — vendor manuals, help
+centers, open textbooks, API specs. It fetches only what the source serves to
+any reader: there is no login/session handling, no paywall traversal, and no
+bot-detection evasion. It is a **polite client**: it identifies itself with a
+`pagespring/<version>` User-Agent (see `.env.example` to override), honors
+`429 Retry-After`, backs off on server errors, paces crawl requests, and caps
+crawl sizes.
+
+It is a *user-invoked, one-manual-at-a-time* archiver — closer to "Save Page
+As" than to an autonomous crawler — so it does not consult `robots.txt`
+(which governs bots that discover URLs on their own; you supply the URL).
+Before mirroring a site, check its terms of use. What you may do with the
+acquired copy (personal RAG corpus, internal search, redistribution) is
+governed by the source's license — the deliverable under `incoming/` stays on
+your machine, and nothing is re-published by this tool.
+
+## Install
+
+```bash
+pip install pagespring
+```
+
+## Quick start
+
+```bash
+pagespring ingest https://docs.tableplus.com   # acquire + normalize → incoming/tableplus/
+pagespring localize <slug>                      # pull a deliverable's images later (resumable; --all)
+pagespring patterns                             # list the source patterns
+pagespring classify <url>                       # which pattern handles a URL (no fetch)
+pagespring status                               # what's been acquired
+```
+
+Deliverables land in `./incoming/<slug>/` under the directory you run from.
+
+## Dev
+
+```bash
+bin/setup   # clone → venv + editable install with dev extras
+bin/test    # pytest
+bin/lint    # ruff check + ruff format --check + mypy (strict)
+```
+
+See [docs/usage.md](docs/usage.md) for the full command set and
+[docs/architecture.md](docs/architecture.md) for the acquire → normalize flow
+and how to add a new source pattern.

pagespring-0.1.0/README.md

@@ -0,0 +1,61 @@
+# pagespring
+
+Acquire and normalize online software manuals into clean, convertible source
+files — the **acquisition front-end to
+[pagespeak](https://github.com/phierceweb/pagespeak)**.
+
+Point it at a manual's URL. A *pattern* recognizes the source type, *acquires*
+the raw pages (stdlib `urllib`), and *normalizes* them into ONE clean
+HTML/markdown file with absolute asset URLs under `incoming/<slug>/`. That clean
+file is the deliverable; converting it into the finished RAG corpus is a separate
+step (pagespeak) that consumes `incoming/` on its own — pagespring never runs it.
+
+Lean by design: `pf-core[cli]` + `beautifulsoup4`, stdlib fetch, no ML stack.
+
+## Intended use
+
+pagespring is for **publicly available documentation** — vendor manuals, help
+centers, open textbooks, API specs. It fetches only what the source serves to
+any reader: there is no login/session handling, no paywall traversal, and no
+bot-detection evasion. It is a **polite client**: it identifies itself with a
+`pagespring/<version>` User-Agent (see `.env.example` to override), honors
+`429 Retry-After`, backs off on server errors, paces crawl requests, and caps
+crawl sizes.
+
+It is a *user-invoked, one-manual-at-a-time* archiver — closer to "Save Page
+As" than to an autonomous crawler — so it does not consult `robots.txt`
+(which governs bots that discover URLs on their own; you supply the URL).
+Before mirroring a site, check its terms of use. What you may do with the
+acquired copy (personal RAG corpus, internal search, redistribution) is
+governed by the source's license — the deliverable under `incoming/` stays on
+your machine, and nothing is re-published by this tool.
+
+## Install
+
+```bash
+pip install pagespring
+```
+
+## Quick start
+
+```bash
+pagespring ingest https://docs.tableplus.com   # acquire + normalize → incoming/tableplus/
+pagespring localize <slug>                      # pull a deliverable's images later (resumable; --all)
+pagespring patterns                             # list the source patterns
+pagespring classify <url>                       # which pattern handles a URL (no fetch)
+pagespring status                               # what's been acquired
+```
+
+Deliverables land in `./incoming/<slug>/` under the directory you run from.
+
+## Dev
+
+```bash
+bin/setup   # clone → venv + editable install with dev extras
+bin/test    # pytest
+bin/lint    # ruff check + ruff format --check + mypy (strict)
+```
+
+See [docs/usage.md](docs/usage.md) for the full command set and
+[docs/architecture.md](docs/architecture.md) for the acquire → normalize flow
+and how to add a new source pattern.

pagespring-0.1.0/pyproject.toml

@@ -0,0 +1,75 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "pagespring"
+version = "0.1.0"
+description = "Acquire and normalize online software manuals into clean, convertible source files — the acquisition front-end to pagespeak."
+readme = "README.md"
+requires-python = ">=3.11"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{ name = "Mike Farr" }]
+keywords = ["manuals", "documentation", "ingest", "acquisition", "rag", "markdown"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Operating System :: OS Independent",
+    "Topic :: Text Processing :: Markup :: Markdown",
+    "Typing :: Typed",
+]
+dependencies = [
+    # Lean by design: stdlib urllib fetches, BeautifulSoup parses, pf-core[cli]
+    # gives the Typer scaffolding + AppConfig + structured logging. No ML stack.
+    "beautifulsoup4>=4.12",
+    # YAML OpenAPI specs (JSON is stdlib); pure-data parser, no transport.
+    "pyyaml>=6.0",
+    "pf-core[cli]~=0.4.1",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.0",
+    "ruff>=0.6",
+    "mypy>=1.10",
+    "pre-commit>=3.7",
+]
+
+[project.scripts]
+pagespring = "pagespring.cli:main"
+
+[project.urls]
+Repository = "https://github.com/phierceweb/pagespring"
+Changelog = "https://github.com/phierceweb/pagespring/blob/main/CHANGELOG.md"
+Issues = "https://github.com/phierceweb/pagespring/issues"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+pagespring = ["py.typed"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"
+
+[tool.ruff.lint]
+select = ["E", "F", "W", "I", "B", "UP", "SIM"]
+ignore = ["E501"]
+
+[tool.ruff.lint.per-file-ignores]
+"src/pagespring/cli.py" = ["B008"]  # Typer uses function calls in argument defaults
+"tests/*" = ["B011"]
+
+[tool.mypy]
+python_version = "3.11"
+strict = true
+ignore_missing_imports = true
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "-ra"

pagespring-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

pagespring-0.1.0/src/pagespring/__init__.py

@@ -0,0 +1,12 @@
+"""pagespring — the lean manual acquisition + normalization layer.
+
+Point it at a manual's URL; it recognizes the source type (a "pattern"),
+*acquires* the raw pages, and *normalizes* them into one clean HTML/markdown
+file with absolute asset URLs under ``incoming/<slug>/``. That clean file is the
+deliverable; *conversion* to RAG markdown is a separate concern (**pagespeak**)
+that consumes ``incoming/`` independently — this package neither runs nor imports it.
+
+This package stays dependency-light (pf-core[cli] + beautifulsoup4).
+"""
+
+__version__ = "0.1.0"

pagespring-0.1.0/src/pagespring/base.py

@@ -0,0 +1,57 @@
+"""The Pattern contract — the unit that ties acquire + normalize + convert
+together for one source type.
+
+A pattern recognizes a family of source URLs (``match``), downloads the raw
+pages (``acquire``), turns them into one clean convertible file with absolute
+asset URLs (``normalize``), and declares the extra ``pagespeak convert`` flags
+its output wants (``convert_recipe``). The conversion engine itself lives in
+pagespeak and is invoked as a subprocess — never imported here.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Literal, Protocol, runtime_checkable
+
+# "html"     -> hand the clean file to pagespeak to convert (markitdown + pipeline)
+# "markdown" -> already the source's canonical clean form (e.g. GitBook per-page .md)
+# "pdf"      -> a downloaded PDF; already a pagespeak input (normalize passthrough)
+SourceKind = Literal["html", "markdown", "pdf"]
+
+
+@dataclass
+class AcquireResult:
+    """What ``acquire`` produced: a local dir of raw pages + how to treat them."""
+
+    raw_dir: Path  # local dir holding the downloaded raw page(s)
+    kind: SourceKind  # whether normalize emits html (for pagespeak) or markdown
+    slug: str  # short id for the source; becomes the output dir name
+    pages: int | None = None  # source units fetched (crawl pages / articles / files); manifest stat
+    title: str | None = None  # human source title for the deliverable heading (falls back to slug)
+
+
+@runtime_checkable
+class Pattern(Protocol):
+    """One source type's acquire/normalize/convert knowledge.
+
+    Implementations are instances (see pagespring/patterns/*); the registry holds
+    one of each. All *source-specific* knowledge (crawl rules, chrome selectors,
+    TOC walking, image-scheme resolution) lives in the pattern — pagespeak stays
+    source-agnostic.
+    """
+
+    name: str
+    convert_recipe: list[str]  # extra flags appended to `pagespeak convert`
+
+    def match(self, url: str) -> bool:
+        """Cheap check (host/path) for whether this pattern handles ``url``."""
+        ...
+
+    def acquire(self, url: str, workdir: Path) -> AcquireResult:
+        """Download the source's raw pages into ``workdir``."""
+        ...
+
+    def normalize(self, acq: AcquireResult, workdir: Path) -> Path:
+        """Turn the raw pages into ONE clean .html/.md (absolute asset URLs)."""
+        ...

pagespring-0.1.0/src/pagespring/cli.py

@@ -0,0 +1,209 @@
+"""pagespring command-line interface (Typer, via pf_core.cli).
+
+Commands:
+    ingest <url>     acquire + normalize ("fix") a manual into incoming/<slug>/
+    localize <slug>  grab an already-ingested deliverable's images (resumable; --all)
+    patterns         list the registered source patterns
+    classify <url>   show which pattern handles a URL (no acquisition)
+    status           list incoming/ deliverables (pattern, pages, size, date, source)
+"""
+
+from datetime import date
+from pathlib import Path
+from urllib.parse import urlsplit
+
+import typer
+from pf_core.cli import create_cli, run_cli
+from pf_core.exceptions import InvalidInputError, PreconditionError
+
+from pagespring import manifest
+from pagespring.config import cfg
+from pagespring.orchestrate import (
+    AcquireError,
+    EmptyOutputError,
+    NoPatternError,
+    localize_images,
+    run_ingest,
+)
+from pagespring.registry import PATTERNS, classify
+
+app = create_cli(
+    "pagespring",
+    help="Find, download, and normalize online software manuals into incoming/.",
+)
+
+
+@app.command()
+def ingest(
+    url: str = typer.Argument(
+        ..., help="Manual URL (e.g. a support.apple.com/guide/<app>/ welcome page)."
+    ),
+    keep_raw: bool = typer.Option(
+        False, "--keep-raw", help="Keep the raw crawl alongside the source in incoming/<slug>/raw."
+    ),
+    download_images: bool = typer.Option(
+        False,
+        "--download-images",
+        help="Download an html/markdown source's images into incoming/<slug>/images/ and re-point refs. No-op for PDFs.",
+    ),
+    if_changed: bool = typer.Option(
+        False,
+        "--if-changed",
+        help="Skip re-staging when the re-fetch normalizes to byte-identical content (the crawl still runs).",
+    ),
+) -> None:
+    """Acquire a manual from URL and normalize it into incoming/<slug>/."""
+    try:
+        result = run_ingest(
+            url, keep_raw=keep_raw, download_images=download_images, if_changed=if_changed
+        )
+    except NoPatternError:
+        typer.echo(
+            f"No pattern matched: {url}\n"
+            "This source needs a new pattern. Run `bin/run patterns` to see the "
+            "registered ones; src/pagespring/patterns/ shows the shape to author one.",
+            err=True,
+        )
+        raise typer.Exit(2) from None
+    except InvalidInputError as exc:
+        typer.echo(str(exc), err=True)
+        raise typer.Exit(2) from None
+    except EmptyOutputError:
+        typer.echo(
+            f"Normalize produced an empty file for {url} — the source may have "
+            "changed shape. Nothing was staged; a previous deliverable in "
+            "incoming/ is untouched.",
+            err=True,
+        )
+        raise typer.Exit(3) from None
+    except AcquireError as exc:
+        typer.echo(
+            f"Fetch failed during acquire: {exc.detail}\n"
+            f"Source: {exc.url}\n"
+            "Nothing was staged. The fetch died mid-acquire — check the URL is "
+            "reachable; re-run to retry.",
+            err=True,
+        )
+        raise typer.Exit(4) from None
+
+    typer.echo(f"pattern  : {result['pattern']}")
+    typer.echo(f"slug     : {result['slug']}")
+    typer.echo(f"incoming : {result['clean']}")
+    if result.get("changed") is False:
+        typer.echo(
+            "status   : unchanged — source matches the existing deliverable, nothing re-staged"
+        )
+        return
+    if result.get("pages") is not None:
+        typer.echo(f"pages    : {result['pages']}")
+    typer.echo(f"size     : {_human_size(result['bytes'])}")
+    if result.get("images"):
+        typer.echo(f"images   : {result['images']} downloaded → images/")
+
+
+@app.command()
+def localize(
+    slug: str = typer.Argument(
+        None, help="Book slug under incoming/ to localize images for (omit when using --all)."
+    ),
+    all_books: bool = typer.Option(
+        False, "--all", help="Localize images for every incoming/<slug>/."
+    ),
+) -> None:
+    """Download an already-ingested deliverable's remote images into images/ and
+    re-point refs — no re-crawl. Resumable: re-run until none remain, so a book too
+    big to localize in one pass finishes across runs."""
+    if all_books:
+        incoming = Path(cfg.INCOMING_DIR)
+        targets = (
+            sorted(p.name for p in incoming.glob("*") if p.is_dir()) if incoming.is_dir() else []
+        )
+    elif slug:
+        targets = [slug]
+    else:
+        typer.echo("Give a slug or --all.", err=True)
+        raise typer.Exit(2)
+
+    for s in targets:
+        try:
+            r = localize_images(s)
+        except PreconditionError as exc:
+            typer.echo(f"skip {s}: {exc}", err=True)
+            continue
+        tail = "done" if r["remaining"] == 0 else f"{r['remaining']} remaining — re-run to continue"
+        typer.echo(f"{s}: +{r['localized']} images (total {r['images_total']}) — {tail}")
+
+
+@app.command()
+def patterns() -> None:
+    """List the registered source patterns and their convert recipes."""
+    for p in PATTERNS:
+        recipe = " ".join(p.convert_recipe) or "(none)"
+        typer.echo(f"{p.name:12} convert-recipe: {recipe}")
+
+
+@app.command("classify")
+def classify_cmd(
+    url: str = typer.Argument(..., help="URL to test against the pattern registry."),
+) -> None:
+    """Show which pattern (if any) handles a URL — no acquisition, no network."""
+    p = classify(url)
+    typer.echo(p.name if p else "(no pattern matched)")
+
+
+def _human_size(n: int) -> str:
+    size = float(n)
+    for unit in ("B", "KB", "MB"):
+        if size < 1024:
+            return f"{size:.0f} {unit}" if unit == "B" else f"{size:.1f} {unit}"
+        size /= 1024
+    return f"{size:.1f} GB"
+
+
+@app.command()
+def status() -> None:
+    """One row per incoming/<slug>/, read from its manifest.json: deliverable,
+    pattern, pages, size, ingest date, and source host. Legacy (pre-manifest)
+    dirs fall back to the deliverable file's own facts. (Conversion into the
+    manuals corpus is pagespeak's job — downstream of this tool, out of its view.)"""
+    incoming = Path(cfg.INCOMING_DIR)
+    slugs = sorted(p for p in incoming.glob("*") if p.is_dir()) if incoming.is_dir() else []
+    if not slugs:
+        typer.echo("(nothing in incoming/ — run `bin/run ingest <url>`)")
+        return
+    for d in slugs:
+        typer.echo(_status_row(d))
+
+
+def _status_row(slug_dir: Path) -> str:
+    """One status line from the slug's manifest; for legacy (pre-manifest) dirs,
+    fall back to the first non-manifest deliverable file's own facts."""
+    m = manifest.read_manifest(slug_dir)
+    if m is not None:
+        deliverable = slug_dir / m["deliverable"]
+        size = deliverable.stat().st_size if deliverable.exists() else m["bytes"]
+        pages = str(m["pages"]) if m["pages"] is not None else "-"
+        host = urlsplit(m["source_url"]).netloc or "-"
+        return (
+            f"{slug_dir.name:24} {m['deliverable']:32} {m['pattern']:14} "
+            f"{pages:>5} {_human_size(size):>9}  {m['ingested_at'][:10]}  {host}"
+        )
+    files = sorted(
+        p for p in slug_dir.iterdir() if p.is_file() and p.name != manifest.MANIFEST_NAME
+    )
+    if not files:
+        return f"{slug_dir.name:24} {'(no clean file)':32} {'-':14} {'-':>5} {'-':>9}  -  -"
+    f = files[0]
+    when = date.fromtimestamp(f.stat().st_mtime).isoformat()
+    return (
+        f"{slug_dir.name:24} {f.name:32} {'-':14} "
+        f"{'-':>5} {_human_size(f.stat().st_size):>9}  {when}  -"
+    )
+
+
+def main() -> None:
+    run_cli(app)
+
+
+if __name__ == "__main__":
+    main()

pagespring-0.1.0/src/pagespring/config.py

@@ -0,0 +1,24 @@
+"""pagespring configuration — a pf_core.config.AppConfig subclass.
+
+All settings are overridable via environment variables / .env.
+"""
+
+from pathlib import Path
+
+from pf_core.config import AppConfig
+
+# src/pagespring/config.py → parents[2] is the project root (editable install).
+_project_root = Path(__file__).resolve().parents[2]
+
+
+class PagespringConfig(AppConfig):
+    """pagespring settings."""
+
+    APP_NAME: str = "pagespring"
+
+    # The deliverable: one incoming/<slug>/ per manual — the clean
+    # acquired+normalized file. A separate step (pagespeak) consumes these.
+    INCOMING_DIR: str = "incoming"
+
+
+cfg = PagespringConfig(env_file=_project_root / ".env")

pagespring-0.1.0/src/pagespring/http.py

@@ -0,0 +1,93 @@
+"""Tiny shared HTTP fetch over the standard library (urllib).
+
+Deliberately no httpx dependency — the acquire step only does plain GETs.
+Provides an identifying User-Agent (override via PAGESPRING_UA), a timeout,
+status-aware retries (permanent 4xx fail fast; 429 honors Retry-After;
+5xx/network errors back off), and a polite inter-request delay for crawls.
+"""
+
+from __future__ import annotations
+
+import time
+import urllib.error
+import urllib.request
+
+from pf_core.utils.env import resolve_str
+
+from pagespring import __version__
+
+_RETRY_AFTER_CAP = 30.0  # seconds — don't let a server park a crawl for minutes
+
+_UA_DEFAULT = f"pagespring/{__version__} (+https://github.com/phierceweb/pagespring)"
+_UA_ENV_VAR = "PAGESPRING_UA"
+
+
+def _ua() -> str:
+    """The identifying default UA, or PAGESPRING_UA for sources that need another."""
+    return resolve_str(None, _UA_ENV_VAR, default=_UA_DEFAULT) or _UA_DEFAULT
+
+
+def _request(url: str) -> urllib.request.Request:
+    return urllib.request.Request(
+        url,
+        headers={
+            "User-Agent": _ua(),
+            "Accept": "*/*",
+            "Accept-Language": "en-US,en;q=0.9",
+        },
+    )
+
+
+def _retry_after(exc: urllib.error.HTTPError, attempt: int) -> float:
+    """Seconds to wait on a 429 — the server's Retry-After (capped) when sane,
+    else the normal backoff."""
+    try:
+        return min(float(exc.headers.get("Retry-After", "")), _RETRY_AFTER_CAP)
+    except ValueError:
+        return 0.5 * (attempt + 1)
+
+
+def _read(url: str, timeout: float, retries: int) -> tuple[str, bytes, str | None]:
+    last: Exception | None = None
+    for attempt in range(retries + 1):
+        try:
+            with urllib.request.urlopen(_request(url), timeout=timeout) as r:
+                return r.geturl(), r.read(), r.headers.get_content_charset()
+        except urllib.error.HTTPError as exc:
+            last = exc
+            if exc.code == 429:
+                if attempt < retries:
+                    time.sleep(_retry_after(exc, attempt))
+            elif 400 <= exc.code < 500 and exc.code != 408:
+                raise  # permanent client error — retrying can't help
+            elif attempt < retries:  # 5xx / 408
+                time.sleep(0.5 * (attempt + 1))
+        except Exception as exc:  # URLError, timeout, connection reset, …
+            last = exc
+            if attempt < retries:
+                time.sleep(0.5 * (attempt + 1))
+    raise last  # type: ignore[misc]
+
+
+def fetch_text(
+    url: str, *, timeout: float = 30, retries: int = 2, encoding: str | None = None
+) -> tuple[str, str]:
+    """Return (final_url, decoded_text) after following redirects.
+
+    Decodes with ``encoding`` when given, else the response's Content-Type
+    charset, else utf-8 — always with replacement, never raising."""
+    final_url, raw, charset = _read(url, timeout, retries)
+    return final_url, raw.decode(encoding or charset or "utf-8", "replace")
+
+
+def fetch_bytes(url: str, *, timeout: float = 180, retries: int = 2) -> tuple[str, bytes]:
+    """Return (final_url, raw_bytes) — for binary downloads (PDFs, archives,
+    images). Longer default timeout than fetch_text: vendor PDFs/doc archives
+    can be tens of MB on slow CDNs."""
+    final_url, raw, _charset = _read(url, timeout, retries)
+    return final_url, raw
+
+
+def polite_sleep(seconds: float = 0.25) -> None:
+    """Sleep between crawl requests to avoid hammering the source."""
+    time.sleep(seconds)