pagespring 0.1.0__py3-none-any.whl

pagespring/__init__.py

@@ -0,0 +1,12 @@
+"""pagespring — the lean manual acquisition + normalization layer.
+
+Point it at a manual's URL; it recognizes the source type (a "pattern"),
+*acquires* the raw pages, and *normalizes* them into one clean HTML/markdown
+file with absolute asset URLs under ``incoming/<slug>/``. That clean file is the
+deliverable; *conversion* to RAG markdown is a separate concern (**pagespeak**)
+that consumes ``incoming/`` independently — this package neither runs nor imports it.
+
+This package stays dependency-light (pf-core[cli] + beautifulsoup4).
+"""
+
+__version__ = "0.1.0"

pagespring/base.py

@@ -0,0 +1,57 @@
+"""The Pattern contract — the unit that ties acquire + normalize + convert
+together for one source type.
+
+A pattern recognizes a family of source URLs (``match``), downloads the raw
+pages (``acquire``), turns them into one clean convertible file with absolute
+asset URLs (``normalize``), and declares the extra ``pagespeak convert`` flags
+its output wants (``convert_recipe``). The conversion engine itself lives in
+pagespeak and is invoked as a subprocess — never imported here.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Literal, Protocol, runtime_checkable
+
+# "html"     -> hand the clean file to pagespeak to convert (markitdown + pipeline)
+# "markdown" -> already the source's canonical clean form (e.g. GitBook per-page .md)
+# "pdf"      -> a downloaded PDF; already a pagespeak input (normalize passthrough)
+SourceKind = Literal["html", "markdown", "pdf"]
+
+
+@dataclass
+class AcquireResult:
+    """What ``acquire`` produced: a local dir of raw pages + how to treat them."""
+
+    raw_dir: Path  # local dir holding the downloaded raw page(s)
+    kind: SourceKind  # whether normalize emits html (for pagespeak) or markdown
+    slug: str  # short id for the source; becomes the output dir name
+    pages: int | None = None  # source units fetched (crawl pages / articles / files); manifest stat
+    title: str | None = None  # human source title for the deliverable heading (falls back to slug)
+
+
+@runtime_checkable
+class Pattern(Protocol):
+    """One source type's acquire/normalize/convert knowledge.
+
+    Implementations are instances (see pagespring/patterns/*); the registry holds
+    one of each. All *source-specific* knowledge (crawl rules, chrome selectors,
+    TOC walking, image-scheme resolution) lives in the pattern — pagespeak stays
+    source-agnostic.
+    """
+
+    name: str
+    convert_recipe: list[str]  # extra flags appended to `pagespeak convert`
+
+    def match(self, url: str) -> bool:
+        """Cheap check (host/path) for whether this pattern handles ``url``."""
+        ...
+
+    def acquire(self, url: str, workdir: Path) -> AcquireResult:
+        """Download the source's raw pages into ``workdir``."""
+        ...
+
+    def normalize(self, acq: AcquireResult, workdir: Path) -> Path:
+        """Turn the raw pages into ONE clean .html/.md (absolute asset URLs)."""
+        ...

pagespring/cli.py

@@ -0,0 +1,209 @@
+"""pagespring command-line interface (Typer, via pf_core.cli).
+
+Commands:
+    ingest <url>     acquire + normalize ("fix") a manual into incoming/<slug>/
+    localize <slug>  grab an already-ingested deliverable's images (resumable; --all)
+    patterns         list the registered source patterns
+    classify <url>   show which pattern handles a URL (no acquisition)
+    status           list incoming/ deliverables (pattern, pages, size, date, source)
+"""
+
+from datetime import date
+from pathlib import Path
+from urllib.parse import urlsplit
+
+import typer
+from pf_core.cli import create_cli, run_cli
+from pf_core.exceptions import InvalidInputError, PreconditionError
+
+from pagespring import manifest
+from pagespring.config import cfg
+from pagespring.orchestrate import (
+    AcquireError,
+    EmptyOutputError,
+    NoPatternError,
+    localize_images,
+    run_ingest,
+)
+from pagespring.registry import PATTERNS, classify
+
+app = create_cli(
+    "pagespring",
+    help="Find, download, and normalize online software manuals into incoming/.",
+)
+
+
+@app.command()
+def ingest(
+    url: str = typer.Argument(
+        ..., help="Manual URL (e.g. a support.apple.com/guide/<app>/ welcome page)."
+    ),
+    keep_raw: bool = typer.Option(
+        False, "--keep-raw", help="Keep the raw crawl alongside the source in incoming/<slug>/raw."
+    ),
+    download_images: bool = typer.Option(
+        False,
+        "--download-images",
+        help="Download an html/markdown source's images into incoming/<slug>/images/ and re-point refs. No-op for PDFs.",
+    ),
+    if_changed: bool = typer.Option(
+        False,
+        "--if-changed",
+        help="Skip re-staging when the re-fetch normalizes to byte-identical content (the crawl still runs).",
+    ),
+) -> None:
+    """Acquire a manual from URL and normalize it into incoming/<slug>/."""
+    try:
+        result = run_ingest(
+            url, keep_raw=keep_raw, download_images=download_images, if_changed=if_changed
+        )
+    except NoPatternError:
+        typer.echo(
+            f"No pattern matched: {url}\n"
+            "This source needs a new pattern. Run `bin/run patterns` to see the "
+            "registered ones; src/pagespring/patterns/ shows the shape to author one.",
+            err=True,
+        )
+        raise typer.Exit(2) from None
+    except InvalidInputError as exc:
+        typer.echo(str(exc), err=True)
+        raise typer.Exit(2) from None
+    except EmptyOutputError:
+        typer.echo(
+            f"Normalize produced an empty file for {url} — the source may have "
+            "changed shape. Nothing was staged; a previous deliverable in "
+            "incoming/ is untouched.",
+            err=True,
+        )
+        raise typer.Exit(3) from None
+    except AcquireError as exc:
+        typer.echo(
+            f"Fetch failed during acquire: {exc.detail}\n"
+            f"Source: {exc.url}\n"
+            "Nothing was staged. The fetch died mid-acquire — check the URL is "
+            "reachable; re-run to retry.",
+            err=True,
+        )
+        raise typer.Exit(4) from None
+
+    typer.echo(f"pattern  : {result['pattern']}")
+    typer.echo(f"slug     : {result['slug']}")
+    typer.echo(f"incoming : {result['clean']}")
+    if result.get("changed") is False:
+        typer.echo(
+            "status   : unchanged — source matches the existing deliverable, nothing re-staged"
+        )
+        return
+    if result.get("pages") is not None:
+        typer.echo(f"pages    : {result['pages']}")
+    typer.echo(f"size     : {_human_size(result['bytes'])}")
+    if result.get("images"):
+        typer.echo(f"images   : {result['images']} downloaded → images/")
+
+
+@app.command()
+def localize(
+    slug: str = typer.Argument(
+        None, help="Book slug under incoming/ to localize images for (omit when using --all)."
+    ),
+    all_books: bool = typer.Option(
+        False, "--all", help="Localize images for every incoming/<slug>/."
+    ),
+) -> None:
+    """Download an already-ingested deliverable's remote images into images/ and
+    re-point refs — no re-crawl. Resumable: re-run until none remain, so a book too
+    big to localize in one pass finishes across runs."""
+    if all_books:
+        incoming = Path(cfg.INCOMING_DIR)
+        targets = (
+            sorted(p.name for p in incoming.glob("*") if p.is_dir()) if incoming.is_dir() else []
+        )
+    elif slug:
+        targets = [slug]
+    else:
+        typer.echo("Give a slug or --all.", err=True)
+        raise typer.Exit(2)
+
+    for s in targets:
+        try:
+            r = localize_images(s)
+        except PreconditionError as exc:
+            typer.echo(f"skip {s}: {exc}", err=True)
+            continue
+        tail = "done" if r["remaining"] == 0 else f"{r['remaining']} remaining — re-run to continue"
+        typer.echo(f"{s}: +{r['localized']} images (total {r['images_total']}) — {tail}")
+
+
+@app.command()
+def patterns() -> None:
+    """List the registered source patterns and their convert recipes."""
+    for p in PATTERNS:
+        recipe = " ".join(p.convert_recipe) or "(none)"
+        typer.echo(f"{p.name:12} convert-recipe: {recipe}")
+
+
+@app.command("classify")
+def classify_cmd(
+    url: str = typer.Argument(..., help="URL to test against the pattern registry."),
+) -> None:
+    """Show which pattern (if any) handles a URL — no acquisition, no network."""
+    p = classify(url)
+    typer.echo(p.name if p else "(no pattern matched)")
+
+
+def _human_size(n: int) -> str:
+    size = float(n)
+    for unit in ("B", "KB", "MB"):
+        if size < 1024:
+            return f"{size:.0f} {unit}" if unit == "B" else f"{size:.1f} {unit}"
+        size /= 1024
+    return f"{size:.1f} GB"
+
+
+@app.command()
+def status() -> None:
+    """One row per incoming/<slug>/, read from its manifest.json: deliverable,
+    pattern, pages, size, ingest date, and source host. Legacy (pre-manifest)
+    dirs fall back to the deliverable file's own facts. (Conversion into the
+    manuals corpus is pagespeak's job — downstream of this tool, out of its view.)"""
+    incoming = Path(cfg.INCOMING_DIR)
+    slugs = sorted(p for p in incoming.glob("*") if p.is_dir()) if incoming.is_dir() else []
+    if not slugs:
+        typer.echo("(nothing in incoming/ — run `bin/run ingest <url>`)")
+        return
+    for d in slugs:
+        typer.echo(_status_row(d))
+
+
+def _status_row(slug_dir: Path) -> str:
+    """One status line from the slug's manifest; for legacy (pre-manifest) dirs,
+    fall back to the first non-manifest deliverable file's own facts."""
+    m = manifest.read_manifest(slug_dir)
+    if m is not None:
+        deliverable = slug_dir / m["deliverable"]
+        size = deliverable.stat().st_size if deliverable.exists() else m["bytes"]
+        pages = str(m["pages"]) if m["pages"] is not None else "-"
+        host = urlsplit(m["source_url"]).netloc or "-"
+        return (
+            f"{slug_dir.name:24} {m['deliverable']:32} {m['pattern']:14} "
+            f"{pages:>5} {_human_size(size):>9}  {m['ingested_at'][:10]}  {host}"
+        )
+    files = sorted(
+        p for p in slug_dir.iterdir() if p.is_file() and p.name != manifest.MANIFEST_NAME
+    )
+    if not files:
+        return f"{slug_dir.name:24} {'(no clean file)':32} {'-':14} {'-':>5} {'-':>9}  -  -"
+    f = files[0]
+    when = date.fromtimestamp(f.stat().st_mtime).isoformat()
+    return (
+        f"{slug_dir.name:24} {f.name:32} {'-':14} "
+        f"{'-':>5} {_human_size(f.stat().st_size):>9}  {when}  -"
+    )
+
+
+def main() -> None:
+    run_cli(app)
+
+
+if __name__ == "__main__":
+    main()

pagespring/config.py

@@ -0,0 +1,24 @@
+"""pagespring configuration — a pf_core.config.AppConfig subclass.
+
+All settings are overridable via environment variables / .env.
+"""
+
+from pathlib import Path
+
+from pf_core.config import AppConfig
+
+# src/pagespring/config.py → parents[2] is the project root (editable install).
+_project_root = Path(__file__).resolve().parents[2]
+
+
+class PagespringConfig(AppConfig):
+    """pagespring settings."""
+
+    APP_NAME: str = "pagespring"
+
+    # The deliverable: one incoming/<slug>/ per manual — the clean
+    # acquired+normalized file. A separate step (pagespeak) consumes these.
+    INCOMING_DIR: str = "incoming"
+
+
+cfg = PagespringConfig(env_file=_project_root / ".env")

pagespring/http.py

@@ -0,0 +1,93 @@
+"""Tiny shared HTTP fetch over the standard library (urllib).
+
+Deliberately no httpx dependency — the acquire step only does plain GETs.
+Provides an identifying User-Agent (override via PAGESPRING_UA), a timeout,
+status-aware retries (permanent 4xx fail fast; 429 honors Retry-After;
+5xx/network errors back off), and a polite inter-request delay for crawls.
+"""
+
+from __future__ import annotations
+
+import time
+import urllib.error
+import urllib.request
+
+from pf_core.utils.env import resolve_str
+
+from pagespring import __version__
+
+_RETRY_AFTER_CAP = 30.0  # seconds — don't let a server park a crawl for minutes
+
+_UA_DEFAULT = f"pagespring/{__version__} (+https://github.com/phierceweb/pagespring)"
+_UA_ENV_VAR = "PAGESPRING_UA"
+
+
+def _ua() -> str:
+    """The identifying default UA, or PAGESPRING_UA for sources that need another."""
+    return resolve_str(None, _UA_ENV_VAR, default=_UA_DEFAULT) or _UA_DEFAULT
+
+
+def _request(url: str) -> urllib.request.Request:
+    return urllib.request.Request(
+        url,
+        headers={
+            "User-Agent": _ua(),
+            "Accept": "*/*",
+            "Accept-Language": "en-US,en;q=0.9",
+        },
+    )
+
+
+def _retry_after(exc: urllib.error.HTTPError, attempt: int) -> float:
+    """Seconds to wait on a 429 — the server's Retry-After (capped) when sane,
+    else the normal backoff."""
+    try:
+        return min(float(exc.headers.get("Retry-After", "")), _RETRY_AFTER_CAP)
+    except ValueError:
+        return 0.5 * (attempt + 1)
+
+
+def _read(url: str, timeout: float, retries: int) -> tuple[str, bytes, str | None]:
+    last: Exception | None = None
+    for attempt in range(retries + 1):
+        try:
+            with urllib.request.urlopen(_request(url), timeout=timeout) as r:
+                return r.geturl(), r.read(), r.headers.get_content_charset()
+        except urllib.error.HTTPError as exc:
+            last = exc
+            if exc.code == 429:
+                if attempt < retries:
+                    time.sleep(_retry_after(exc, attempt))
+            elif 400 <= exc.code < 500 and exc.code != 408:
+                raise  # permanent client error — retrying can't help
+            elif attempt < retries:  # 5xx / 408
+                time.sleep(0.5 * (attempt + 1))
+        except Exception as exc:  # URLError, timeout, connection reset, …
+            last = exc
+            if attempt < retries:
+                time.sleep(0.5 * (attempt + 1))
+    raise last  # type: ignore[misc]
+
+
+def fetch_text(
+    url: str, *, timeout: float = 30, retries: int = 2, encoding: str | None = None
+) -> tuple[str, str]:
+    """Return (final_url, decoded_text) after following redirects.
+
+    Decodes with ``encoding`` when given, else the response's Content-Type
+    charset, else utf-8 — always with replacement, never raising."""
+    final_url, raw, charset = _read(url, timeout, retries)
+    return final_url, raw.decode(encoding or charset or "utf-8", "replace")
+
+
+def fetch_bytes(url: str, *, timeout: float = 180, retries: int = 2) -> tuple[str, bytes]:
+    """Return (final_url, raw_bytes) — for binary downloads (PDFs, archives,
+    images). Longer default timeout than fetch_text: vendor PDFs/doc archives
+    can be tens of MB on slow CDNs."""
+    final_url, raw, _charset = _read(url, timeout, retries)
+    return final_url, raw
+
+
+def polite_sleep(seconds: float = 0.25) -> None:
+    """Sleep between crawl requests to avoid hammering the source."""
+    time.sleep(seconds)

pagespring/images.py

@@ -0,0 +1,125 @@
+"""Optional image localizer for HTML/markdown ingests.
+
+Downloads a deliverable's remote images into a sibling ``images/`` dir and
+re-points the refs at them — for a self-contained ``incoming/<slug>/``, and to
+capture images behind expiring or tokened URLs (e.g. GitBook's
+``?alt=media&token=…``) while they still resolve.
+
+Opt-in via ``bin/run ingest --download-images`` or ``bin/run localize``.
+Stdlib fetch only (``pagespring.http``).
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from urllib.parse import urlparse
+
+from pf_core.log import get_logger
+
+from pagespring import http
+
+log = get_logger(__name__)
+
+# Markdown  ![alt](http…)  and HTML  <img … src="http…">.
+_MD_IMG_RE = re.compile(r"!\[[^\]]*\]\((https?://[^)\s]+)\)")
+_HTML_IMG_RE = re.compile(r"""<img\b[^>]*?\bsrc=["'](https?://[^"']+)["']""", re.IGNORECASE)
+
+_IMG_EXTS = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".bmp", ".tif", ".tiff"}
+_MAGIC = (
+    (b"\x89PNG\r\n\x1a\n", ".png"),
+    (b"\xff\xd8\xff", ".jpg"),
+    (b"GIF87a", ".gif"),
+    (b"GIF89a", ".gif"),
+    (b"<svg", ".svg"),
+    (b"<?xml", ".svg"),
+)
+
+
+def _ext_for(url: str, data: bytes) -> str:
+    suffix = Path(urlparse(url).path).suffix.lower()
+    if suffix in _IMG_EXTS:
+        return ".jpg" if suffix == ".jpeg" else (".tiff" if suffix == ".tif" else suffix)
+    head = data[:16]
+    for magic, ext in _MAGIC:
+        if head.startswith(magic):
+            return ext
+    if head[:4] == b"RIFF" and b"WEBP" in data[:16]:
+        return ".webp"
+    return ".img"
+
+
+def _name_for(url: str, data: bytes, used: set[str]) -> str:
+    base = Path(urlparse(url).path).name
+    stem = re.sub(r"\.[A-Za-z0-9]+$", "", base)  # drop ext; we set our own
+    stem = re.sub(r"[^A-Za-z0-9._-]+", "-", stem).strip("-") or "image"
+    ext = _ext_for(url, data)
+    name = f"{stem}{ext}"
+    i = 2
+    while name in used:
+        name = f"{stem}-{i}{ext}"
+        i += 1
+    used.add(name)
+    return name
+
+
+def _remote_image_urls(text: str) -> list[str]:
+    """Distinct remote (http/https) image refs in ``text``, in first-seen order."""
+    urls: list[str] = []
+    seen: set[str] = set()
+    for rx in (_MD_IMG_RE, _HTML_IMG_RE):
+        for u in rx.findall(text):
+            if u not in seen:
+                seen.add(u)
+                urls.append(u)
+    return urls
+
+
+def count_remote_images(doc_path: Path) -> int:
+    """Distinct remote image refs still in ``doc_path`` (0 ⇒ fully localized) — lets
+    a caller know whether another ``download_images`` pass is needed."""
+    return len(_remote_image_urls(doc_path.read_text(encoding="utf-8")))
+
+
+def download_images(doc_path: Path, images_dir: Path, *, checkpoint_every: int = 50) -> int:
+    """Download the doc's remote images into ``images_dir`` and re-point refs to
+    ``images/<name>``. Returns the count downloaded this run; unfetchable refs are
+    left untouched (logged).
+
+    Resumable: each image is re-pointed in the deliverable the moment it lands (the
+    file IS the progress ledger — finished refs are ``images/<name>``, pending ones
+    stay remote), and the doc is checkpointed every ``checkpoint_every`` images, so
+    a run killed partway keeps what it localized. ``used`` names are seeded from
+    ``images_dir`` so a resumed run can't clobber a prior run's files. Re-run until
+    ``count_remote_images`` returns 0 (how big books beat a per-run time cap).
+    """
+    text = doc_path.read_text(encoding="utf-8")
+    urls = _remote_image_urls(text)
+    if not urls:
+        return 0
+
+    images_dir.mkdir(parents=True, exist_ok=True)
+    # Seed from a prior run's files so a resumed download can't clobber them.
+    used: set[str] = {p.name for p in images_dir.iterdir() if p.is_file()}
+    saved = 0
+    since_checkpoint = 0
+    # Longest URL first so one ref can't be a prefix of another when we re-point.
+    for u in sorted(urls, key=len, reverse=True):
+        try:
+            _f, data = http.fetch_bytes(u)
+        except Exception as exc:
+            log.warning("images.fetch_error", url=u, error=str(exc))
+            continue
+        name = _name_for(u, data, used)
+        (images_dir / name).write_bytes(data)
+        text = text.replace(u, f"images/{name}")  # re-point now: the file is the ledger
+        saved += 1
+        since_checkpoint += 1
+        if since_checkpoint >= checkpoint_every:
+            doc_path.write_text(text, encoding="utf-8")
+            since_checkpoint = 0
+        http.polite_sleep()
+
+    doc_path.write_text(text, encoding="utf-8")
+    log.info("images.download", doc=str(doc_path), found=len(urls), saved=saved)
+    return saved

pagespring/manifest.py

@@ -0,0 +1,98 @@
+"""The per-slug ``manifest.json`` — the provenance record written beside each
+``incoming/<slug>/`` deliverable.
+
+It records where a manual came from, which pattern acquired it, the downstream
+pagespeak ``convert_recipe`` hint, and a content hash — so the hand-off to
+pagespeak is self-describing, and so ``ingest --if-changed`` can tell whether a
+re-fetch produced anything new. Pure stdlib; no network, no pattern machinery.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+from typing import TypedDict
+
+from pagespring import __version__
+
+MANIFEST_NAME = "manifest.json"
+SCHEMA_VERSION = 1
+
+
+class Manifest(TypedDict):
+    """The on-disk shape of ``incoming/<slug>/manifest.json`` (schema v1)."""
+
+    schema_version: int
+    pagespring_version: str
+    source_url: str
+    pattern: str
+    slug: str
+    kind: str
+    deliverable: str
+    convert_recipe: list[str]
+    pages: int | None
+    bytes: int
+    sha256: str
+    images: int
+    ingested_at: str
+
+
+def sha256_file(path: Path) -> str:
+    """Hex SHA-256 of ``path``'s bytes (the deliverable's content identity)."""
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def build_manifest(
+    *,
+    source_url: str,
+    pattern: str,
+    slug: str,
+    kind: str,
+    deliverable: str,
+    convert_recipe: list[str],
+    pages: int | None,
+    size_bytes: int,
+    sha256: str,
+    images: int,
+    ingested_at: str,
+) -> Manifest:
+    """Assemble a manifest from one ingest's facts (stamps schema + version)."""
+    return {
+        "schema_version": SCHEMA_VERSION,
+        "pagespring_version": __version__,
+        "source_url": source_url,
+        "pattern": pattern,
+        "slug": slug,
+        "kind": kind,
+        "deliverable": deliverable,
+        "convert_recipe": convert_recipe,
+        "pages": pages,
+        "bytes": size_bytes,
+        "sha256": sha256,
+        "images": images,
+        "ingested_at": ingested_at,
+    }
+
+
+def write_manifest(slug_dir: Path, manifest: Manifest) -> Path:
+    """Write ``manifest`` as pretty JSON to ``slug_dir/manifest.json``; return it."""
+    path = slug_dir / MANIFEST_NAME
+    path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8")
+    return path
+
+
+def read_manifest(slug_dir: Path) -> Manifest | None:
+    """Read ``slug_dir/manifest.json``; ``None`` if absent or unparseable.
+
+    Tolerant by design: a legacy slug dir (pre-manifest) or a corrupt file must
+    not crash ``status`` or ``--if-changed`` — they treat ``None`` as "no record".
+    """
+    path = slug_dir / MANIFEST_NAME
+    if not path.exists():
+        return None
+    try:
+        data: Manifest = json.loads(path.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError):
+        return None
+    return data