getdocs 0.1.0__py3-none-any.whl

getdocs/engine.py

@@ -0,0 +1,418 @@
+"""Engine: shallow Scrapy glue around the deep modules.
+
+Runs one Crawl per process via CrawlerProcess — the Twisted reactor starts
+once and never restarts (ADR-0002), which is why the future API service
+spawns this as a subprocess per Crawl.
+"""
+
+import asyncio
+import json
+import posixpath
+import sys
+from dataclasses import replace
+from datetime import datetime, timezone
+from pathlib import Path
+from urllib.parse import urljoin, urlsplit
+
+import scrapy
+from scrapy.crawler import CrawlerProcess
+from scrapy.downloadermiddlewares.retry import get_retry_request
+from scrapy.exceptions import CloseSpider, IgnoreRequest
+from scrapy.http import TextResponse
+from scrapy.spidermiddlewares.httperror import HttpError
+
+from getdocs.config import CrawlConfig
+from getdocs.extract import extract_page, is_shell
+from getdocs.identity import build_user_agent
+from getdocs.navharvest import harvest_nav, merge_harvests
+from getdocs.output import AssetStore, FileTreeWriter, JsonlWriter, PageRecord, relink_pages
+from getdocs.scope import Scope
+from getdocs.sitemap import parse_robots_sitemaps, parse_sitemap_xml
+from getdocs.urlnorm import normalize
+
+
+def state_file_for(config: CrawlConfig) -> Path:
+    return config.output_dir / ".getdocs" / "crawl-state.json"
+
+
+def playwright_available() -> bool:
+    import importlib.util
+
+    return importlib.util.find_spec("scrapy_playwright") is not None
+
+
+_SHELL_LATCH_THRESHOLD = 2  # Shells on one host before it renders everything
+
+
+class RetryAfterMiddleware:
+    """Retry 429 responses no sooner than the server's Retry-After asks.
+
+    Scrapy's stock RetryMiddleware retries 429 immediately, which is exactly
+    what a rate-limiting server is telling us not to do — so 429 is removed
+    from its codes and handled here with an async sleep (asyncio reactor).
+    """
+
+    async def process_response(self, request, response, spider):
+        if response.status != 429:
+            return response
+        retry = get_retry_request(request, spider=spider, reason="429 Too Many Requests")
+        if retry is None:
+            return response  # retries exhausted; falls through to the errback
+        try:
+            delay = float(response.headers.get("Retry-After", b"1"))
+        except ValueError:
+            delay = 1.0
+        await asyncio.sleep(delay)
+        return retry
+
+
+class _CrawlSpider(scrapy.Spider):
+    name = "getdocs"
+
+    def __init__(
+        self, config: CrawlConfig, writer, outcome: dict, resume_state: dict | None = None,
+        render_enabled: bool = False, **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.config = config
+        self.writer = writer
+        self.outcome = outcome
+        self.resume_state = resume_state
+        self.render_enabled = render_enabled
+        self.shell_hosts: dict[str, int] = {}
+        # Asset download bookkeeping (--download-media). A Page's write is
+        # deferred until its Assets resolve, so failed/oversized Assets can
+        # keep their absolute URLs in the written markdown.
+        self.asset_store = AssetStore(config.output_dir)
+        self.asset_results: dict[str, str | None] = {}
+        self.asset_inflight: set[str] = set()
+        self.pending_pages: list[dict] = []
+        self.media_cap_bytes = int(config.media_max_size * 1024 * 1024)
+        # Frontier bookkeeping, persisted across runs (see closed()).
+        # pending maps a yielded URL to its hop count; an entry is removed
+        # only once the URL is written or errored, so an interruption never
+        # loses in-flight work.
+        self.pending: dict[str, int] = {}
+        self.scope = Scope.from_seeds(
+            config.seeds,
+            allow_backward=config.allow_backward,
+            allow_subdomains=config.allow_subdomains,
+            include_paths=config.include_paths,
+            exclude_paths=config.exclude_paths,
+        )
+        self.follow_links = config.sitemap != "only"
+        self.enqueued: set[str] = set(resume_state["enqueued"]) if resume_state else set()
+        self.written: set[str] = set(resume_state["written"]) if resume_state else set()
+        self.sitemaps_fetched: set[str] = set()
+        self.crawl_sequence: list[str] = outcome["crawl_sequence"]
+
+    # -- discovery ---------------------------------------------------------
+
+    async def start(self):
+        if self.config.sitemap != "off":
+            roots = {f"{urlsplit(s).scheme}://{urlsplit(s).netloc}" for s in self.config.seeds}
+            for root in sorted(roots):
+                yield scrapy.Request(urljoin(root, "/robots.txt"), callback=self.parse_robots)
+                for request in self._sitemap_requests([urljoin(root, "/sitemap.xml")]):
+                    yield request
+        if self.resume_state:
+            for url, hops in self.resume_state["pending"].items():
+                self.pending[url] = hops
+                yield self._page_request(url, hops=hops)
+        elif self.config.sitemap != "only":
+            for seed in self.config.seeds:
+                self.enqueued.add(normalize(seed))
+                self.pending[seed] = 0
+                yield self._page_request(seed, hops=0)
+
+    def parse_robots(self, response):
+        yield from self._sitemap_requests(parse_robots_sitemaps(response.text))
+
+    def parse_sitemap(self, response):
+        page_urls, nested = parse_sitemap_xml(response.text)
+        yield from self._sitemap_requests(nested)
+        for url in page_urls:
+            # Sitemap-discovered Pages are depth-0 seeds: Scope still gates
+            # them, but --depth never excludes them.
+            yield from self._enqueue_page(url, hops=0)
+
+    # -- fetching ----------------------------------------------------------
+
+    def parse_page(self, response):
+        rendered = response.meta.get("playwright", False)
+        shellish = (
+            not rendered and isinstance(response, TextResponse) and is_shell(response.text)
+        )
+        if shellish and self.config.render == "auto" and self.render_enabled:
+            # Escalate: re-fetch through the browser. The pending entry
+            # survives until the rendered version is written.
+            host = urlsplit(response.url).netloc
+            self.shell_hosts[host] = self.shell_hosts.get(host, 0) + 1
+            yield scrapy.Request(
+                response.request.url,
+                callback=self.parse_page,
+                errback=self.on_page_error,
+                meta={"hops": response.meta["hops"], "playwright": True},
+                dont_filter=True,
+            )
+            return
+
+        norm = normalize(response.url)
+        if norm not in self.written:
+            if self.config.limit and len(self.written) >= self.config.limit:
+                self.outcome["truncated"] = True
+                raise CloseSpider("page limit reached")
+            self.written.add(norm)
+            extracted = extract_page(response.text, response.url, selector=self.config.selector)
+            record = PageRecord(
+                url=response.url,
+                title=extracted.title,
+                markdown=extracted.markdown,
+                status=response.status,
+                crawled_at=datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+                canonical=extracted.canonical,
+                html=response.text if self.config.keep_html else None,
+            )
+            if shellish:
+                # Rendering is off or unavailable: written as-is, flagged.
+                self.outcome["shells"].append(response.url)
+            self.crawl_sequence.append(response.url)
+            self.outcome["harvests"].append(
+                {"page": response.url, **harvest_nav(response.text, response.url)}
+            )
+
+            assets = list(dict.fromkeys(extracted.assets)) if self.config.download_media else []
+            waiting = set()
+            for asset_url in assets:
+                if asset_url in self.asset_results:
+                    continue
+                waiting.add(asset_url)
+                if asset_url not in self.asset_inflight:
+                    self.asset_inflight.add(asset_url)
+                    yield scrapy.Request(
+                        asset_url,
+                        callback=self.on_asset,
+                        errback=self.on_asset_error,
+                        meta={"download_maxsize": self.media_cap_bytes},
+                        dont_filter=True,
+                    )
+            if waiting:
+                self.pending_pages.append(
+                    {"record": record, "assets": assets, "waiting": waiting}
+                )
+            else:
+                self._finalize_page(record, assets)
+        self.pending.pop(response.request.url, None)
+
+        if not self.follow_links or not isinstance(response, TextResponse):
+            return
+        hops = response.meta["hops"]
+        if self.config.depth and hops + 1 > self.config.depth:
+            return
+        for href in response.css("a::attr(href)").getall():
+            yield from self._enqueue_page(response.urljoin(href.strip()), hops=hops + 1)
+
+    # -- assets (--download-media; Scope never applies, ADR-0005) -----------
+
+    def on_asset(self, response):
+        relpath = self.asset_store.save(response.request.url, response.body)
+        self._resolve_asset(response.request.url, relpath)
+
+    def on_asset_error(self, failure):
+        url = failure.request.url
+        reason = (failure.getErrorMessage() or failure.type.__name__)[:200]
+        self.outcome["media_skipped"].append({"url": url, "reason": reason})
+        self._resolve_asset(url, None)
+
+    def _resolve_asset(self, url: str, relpath: str | None) -> None:
+        self.asset_results[url] = relpath
+        self.asset_inflight.discard(url)
+        still_waiting = []
+        for page in self.pending_pages:
+            page["waiting"].discard(url)
+            if page["waiting"]:
+                still_waiting.append(page)
+            else:
+                self._finalize_page(page["record"], page["assets"])
+        self.pending_pages = still_waiting
+
+    def _finalize_page(self, record: PageRecord, assets: list[str]) -> None:
+        markdown = record.markdown
+        for url in assets:
+            relpath = self.asset_results.get(url)
+            if relpath:
+                markdown = markdown.replace(url, self._asset_link(record.url, relpath))
+        if markdown is not record.markdown:
+            record = replace(record, markdown=markdown)
+        self.writer.write_page(record)
+        self._progress()
+
+    def _asset_link(self, page_url: str, asset_relpath: str) -> str:
+        """files mode: relative from the page's .md location; jsonl mode:
+        root-relative (records have no on-disk location)."""
+        if isinstance(self.writer, FileTreeWriter):
+            page_rel = self.writer.path_for(page_url).relative_to(self.writer.output_dir)
+            return posixpath.relpath(asset_relpath, start=page_rel.parent.as_posix())
+        return asset_relpath
+
+    def on_page_error(self, failure):
+        self.pending.pop(failure.request.url, None)
+        if failure.check(HttpError):
+            response = failure.value.response
+            error = {"url": response.url, "status": response.status, "reason": f"HTTP {response.status}"}
+        elif failure.check(IgnoreRequest):
+            # HttpError subclasses IgnoreRequest, so this arm only sees true
+            # filtering — robots.txt telling us not to fetch.
+            self.outcome["skipped"].append(
+                {"url": failure.request.url, "reason": "robots.txt"}
+            )
+            self._progress()
+            return
+        else:
+            error = {"url": failure.request.url, "status": None, "reason": failure.type.__name__}
+        self.outcome["errors"].append(error)
+        self._progress()
+
+    # -- helpers -----------------------------------------------------------
+
+    def _progress(self):
+        done = len(self.written) + len(self.outcome["errors"])
+        print(
+            f"[getdocs] pages={len(self.written)} "
+            f"pending={max(len(self.enqueued) - done, 0)} "
+            f"errors={len(self.outcome['errors'])}",
+            file=sys.stderr,
+            flush=True,
+        )
+
+    def _page_request(self, url: str, hops: int) -> scrapy.Request:
+        meta = {"hops": hops}
+        if self._should_render(url):
+            meta["playwright"] = True
+        return scrapy.Request(
+            url, callback=self.parse_page, errback=self.on_page_error, meta=meta
+        )
+
+    def _should_render(self, url: str) -> bool:
+        if not self.render_enabled:
+            return False
+        if self.config.render == "always":
+            return True
+        host = urlsplit(url).netloc
+        return self.shell_hosts.get(host, 0) >= _SHELL_LATCH_THRESHOLD
+
+    def _enqueue_page(self, url: str, hops: int):
+        if not self.scope.allows(url):
+            return
+        norm = normalize(url)
+        if norm in self.enqueued:
+            return
+        self.enqueued.add(norm)
+        self.pending[url] = hops
+        yield self._page_request(url, hops=hops)
+
+    def _sitemap_requests(self, urls: list[str]):
+        for url in urls:
+            if url not in self.sitemaps_fetched:
+                self.sitemaps_fetched.add(url)
+                yield scrapy.Request(url, callback=self.parse_sitemap)
+
+    def closed(self, reason):
+        # Pages still waiting on Assets write now with whatever resolved;
+        # unresolved Assets keep their absolute URLs.
+        for page in self.pending_pages:
+            self._finalize_page(page["record"], page["assets"])
+        self.pending_pages = []
+        state_file = state_file_for(self.config)
+        state_file.parent.mkdir(parents=True, exist_ok=True)
+        state_file.write_text(
+            json.dumps(
+                {
+                    "seeds": self.config.seeds,
+                    "enqueued": sorted(self.enqueued),
+                    "written": sorted(self.written),
+                    "pending": self.pending,
+                    "errors": self.outcome["errors"],
+                    "skipped": self.outcome["skipped"],
+                    "shells": self.outcome["shells"],
+                    "crawl_sequence": self.crawl_sequence,
+                    "harvests": self.outcome["harvests"],
+                    "media_skipped": self.outcome["media_skipped"],
+                }
+            )
+        )
+
+
+def run_crawl(config: CrawlConfig) -> int:
+    """Run a Crawl to completion; returns the number of Pages produced."""
+    if config.format == "jsonl":
+        writer = JsonlWriter(sys.stdout)
+    else:
+        writer = FileTreeWriter(config.output_dir)
+    resume_state = None
+    if config.resume:
+        resume_state = json.loads(state_file_for(config).read_text())
+        writer.page_count = len(resume_state["written"])
+    outcome = {
+        "errors": list(resume_state["errors"]) if resume_state else [],
+        "skipped": list(resume_state["skipped"]) if resume_state else [],
+        "shells": list(resume_state.get("shells", [])) if resume_state else [],
+        "harvests": list(resume_state.get("harvests", [])) if resume_state else [],
+        "media_skipped": list(resume_state.get("media_skipped", [])) if resume_state else [],
+        # Shared with the spider, which appends as Pages are written.
+        "crawl_sequence": list(resume_state.get("crawl_sequence", [])) if resume_state else [],
+        "truncated": False,  # recomputed by this run: a resumed Crawl may finish
+    }
+    render_enabled = config.render != "never" and playwright_available()
+    if config.render != "never" and not render_enabled:
+        print(
+            "note: scrapy-playwright is not installed — JS rendering disabled; "
+            "Shell pages will be written as-is and flagged in the Manifest",
+            file=sys.stderr,
+        )
+    settings = {
+        "LOG_LEVEL": "ERROR",
+        # Identify honestly as getdocs (not generic Scrapy); robots.txt matching
+        # uses this UA too. --contact appends the user's email/URL.
+        "USER_AGENT": build_user_agent(config.contact, config.user_agent),
+        "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
+        "RETRY_TIMES": 2,
+        # 429 is handled by RetryAfterMiddleware, which honors Retry-After.
+        "RETRY_HTTP_CODES": [500, 502, 503, 504, 522, 524, 408],
+        "DOWNLOADER_MIDDLEWARES": {RetryAfterMiddleware: 560},
+        "ROBOTSTXT_OBEY": not config.ignore_robots,
+        "DOWNLOAD_DELAY": config.delay,
+        "AUTOTHROTTLE_ENABLED": config.delay > 0,
+        "AUTOTHROTTLE_START_DELAY": config.delay or 1.0,
+        "CONCURRENT_REQUESTS_PER_DOMAIN": config.concurrency,
+    }
+    if render_enabled:
+        settings["DOWNLOAD_HANDLERS"] = {
+            "http": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",
+            "https": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",
+        }
+        settings["TWISTED_REACTOR"] = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
+    process = CrawlerProcess(settings=settings, install_root_handler=False)
+    process.crawl(
+        _CrawlSpider,
+        config=config,
+        writer=writer,
+        outcome=outcome,
+        resume_state=resume_state,
+        render_enabled=render_enabled,
+    )
+    process.start()
+    if isinstance(writer, FileTreeWriter):
+        relink_pages(writer, outcome["crawl_sequence"])
+    nav, reading_order = merge_harvests(outcome["harvests"], outcome["crawl_sequence"])
+    writer.write_manifest(
+        seeds=config.seeds,
+        errors=outcome["errors"],
+        truncated=outcome["truncated"],
+        skipped=outcome["skipped"],
+        shells=outcome["shells"],
+        nav=nav,
+        reading_order=reading_order,
+        media_skipped=outcome["media_skipped"],
+    )
+    return writer.page_count

getdocs/extract.py

@@ -0,0 +1,190 @@
+"""Extract: HTML in, (title, markdown, canonical) out.
+
+Selector-first pipeline: a user-supplied selector wins, then the known
+content containers of common docs generators, then semantic candidates.
+Readability extraction (trafilatura) is a last resort for pages with no
+recognizable content root — docs sites are structured, so exploiting
+that structure beats statistical extraction (and never eats code blocks).
+"""
+
+import re
+from dataclasses import dataclass
+from urllib.parse import urljoin, urlsplit
+
+from bs4 import BeautifulSoup
+from markdownify import MarkdownConverter, markdownify
+
+
+@dataclass(frozen=True)
+class ExtractedPage:
+    title: str
+    markdown: str
+    canonical: str | None = None
+    assets: tuple[str, ...] = ()  # absolute URLs of referenced images/documents
+
+
+_DOC_EXTENSIONS = (
+    ".pdf", ".zip", ".tar.gz", ".tgz", ".7z", ".dmg", ".pkg", ".msi", ".exe", ".whl",
+)
+
+
+_GENERATOR_SELECTORS = [
+    "div.theme-doc-markdown",      # Docusaurus
+    "article.md-content__inner",   # MkDocs Material
+    'div.body[role="main"]',       # Sphinx
+    "#content-area",               # Mintlify
+]
+_SEMANTIC_SELECTORS = ["main", "article", '[role="main"]']
+
+_NOISE_TAGS = ("nav", "aside", "header", "footer", "script", "style", "noscript")
+_NOISE_CLASS_RE = re.compile(r"breadcrumb|table-of-contents|(^|[-_])toc($|[-_])")
+
+
+def _code_language(el) -> str:
+    """Language hint for a <pre> block: language-x on it or its <code>
+    children, or Sphinx-style highlight-x on an ancestor."""
+    candidates = [el, *el.find_all("code"), *el.parents]
+    for node in candidates:
+        for cls in node.get("class") or []:
+            if cls.startswith("language-"):
+                return cls.removeprefix("language-")
+            if cls.startswith("highlight-"):
+                lang = cls.removeprefix("highlight-")
+                if lang not in ("default", "notranslate"):
+                    return lang
+    return ""
+
+
+_converter = MarkdownConverter(heading_style="ATX", code_language_callback=_code_language)
+
+
+# Known icon shapes (Material Design icon path prefixes, as emitted by
+# mkdocs-material twemoji spans). Inline SVGs carry no text, so without
+# this, checkmark columns in comparison tables extract as empty cells.
+_SVG_GLYPHS = {
+    "M21 7 9 19l-5.5-5.5": "✓",  # mdi-check
+    "M9 20.42 2.79 14.21": "✓",  # mdi-check-bold
+    "M19 6.41 17.59 5 12 10.59": "✗",  # mdi-close
+    "M20 6.91 17.09 4 12 9.09": "✗",  # mdi-close-thick
+}
+
+
+def _svg_to_text(root) -> None:
+    for svg in root.find_all("svg"):
+        label = svg.get("aria-label")
+        if not label:
+            title = svg.find("title")
+            label = title.get_text(strip=True) if title else None
+        if not label:
+            path = svg.find("path")
+            d = (path.get("d") or "") if path else ""
+            label = next((g for prefix, g in _SVG_GLYPHS.items() if d.startswith(prefix)), None)
+        if label:
+            svg.replace_with(label)
+
+
+def _absolutize_urls(root, page_url: str) -> list[str]:
+    """Rewrite hrefs/srcs absolute against the page URL — relative values
+    would point at nothing in the output tree (hotlink default, ADR-0005).
+    Returns the Asset URLs found: images, then document downloads."""
+    images, documents = [], []
+    for tag in root.find_all(href=True):
+        tag["href"] = urljoin(page_url, tag["href"])
+        if tag.name == "a" and urlsplit(tag["href"]).path.lower().endswith(_DOC_EXTENSIONS):
+            documents.append(tag["href"])
+    for tag in root.find_all(src=True):
+        tag["src"] = urljoin(page_url, tag["src"])
+        if tag.name == "img":
+            images.append(tag["src"])
+    for tag in root.find_all(srcset=True):
+        tag["srcset"] = ", ".join(
+            " ".join([urljoin(page_url, part.strip().split()[0]), *part.strip().split()[1:]])
+            for part in tag["srcset"].split(",")
+            if part.strip()
+        )
+    return images + documents
+
+
+def _strip_noise(root) -> None:
+    for tag in root.find_all(_NOISE_TAGS):
+        tag.decompose()
+    doomed = [
+        el
+        for el in root.find_all(True)
+        if any(_NOISE_CLASS_RE.search(cls) for cls in el.get("class") or [])
+    ]
+    for el in doomed:
+        el.decompose()
+
+
+def _find_content_root(soup, selector: str | None):
+    if selector:
+        root = soup.select_one(selector)
+        if root is not None:
+            return root
+    for sel in [*_GENERATOR_SELECTORS, *_SEMANTIC_SELECTORS]:
+        root = soup.select_one(sel)
+        if root is not None:
+            return root
+    return None
+
+
+def _readability_markdown(html: str) -> str | None:
+    import trafilatura
+
+    return trafilatura.extract(html, output_format="markdown", include_tables=True)
+
+
+_ROOT_DIV_IDS = ["root", "app", "__next", "___gatsby", "__nuxt"]
+
+
+def is_shell(html: str) -> bool:
+    """True when a response is an unhydrated client-side app frame rather
+    than real content — the signal that triggers render escalation."""
+    soup = BeautifulSoup(html, "html.parser")
+    body = soup.body
+    if body is None:
+        return False
+
+    noscript_warning = any(
+        "javascript" in ns.get_text().lower() for ns in body.find_all("noscript")
+    )
+    has_scripts = bool(soup.find("script"))
+
+    for tag in body.find_all(["script", "noscript", "style", "template"]):
+        tag.decompose()
+    text = body.get_text(strip=True)
+    if len(text) > 200:
+        return False  # plenty of real text — not a shell, whatever else it has
+
+    empty_root = any(
+        not div.get_text(strip=True) for div in body.find_all("div", id=_ROOT_DIV_IDS)
+    )
+    return empty_root or noscript_warning or (has_scripts and len(text) < 30)
+
+
+def extract_page(html: str, url: str, selector: str | None = None) -> ExtractedPage:
+    soup = BeautifulSoup(html, "html.parser")
+
+    title = soup.title.get_text(strip=True) if soup.title else None
+    if not title:
+        og_title = soup.find("meta", property="og:title")
+        title = og_title.get("content") if og_title else None
+    title = title or url
+
+    canonical_link = soup.find("link", rel="canonical")
+    canonical = canonical_link.get("href") if canonical_link else None
+
+    root = _find_content_root(soup, selector)
+    assets: list[str] = []
+    if root is not None:
+        _strip_noise(root)
+        _svg_to_text(root)
+        assets = _absolutize_urls(root, url)
+        markdown = _converter.convert(str(root)).strip()
+    else:
+        markdown = (_readability_markdown(html) or markdownify(str(soup.body or soup))).strip()
+
+    return ExtractedPage(
+        title=title, markdown=markdown, canonical=canonical, assets=tuple(assets)
+    )

getdocs/identity.py

@@ -0,0 +1,32 @@
+"""How getdocs identifies itself to the sites it fetches.
+
+One honest, descriptive User-Agent for every request — both the pre-crawl
+source check (`source.py`) and the Scrapy crawl (`engine.py`) — so a site
+operator reading their logs can tell it's getdocs and, when the user opts in
+with `--contact`, reach whoever is crawling. Identifying yourself is crawling
+etiquette (RFC 9309), not a hard requirement, so `contact` stays optional.
+"""
+
+import importlib.metadata
+
+PROJECT_URL = "https://github.com/jonbakerfish/getdocs"
+
+
+def _version() -> str:
+    try:
+        return importlib.metadata.version("getdocs")
+    except importlib.metadata.PackageNotFoundError:
+        return "0.0.0"
+
+
+def build_user_agent(contact: str | None = None, override: str | None = None) -> str:
+    """The User-Agent getdocs sends.
+
+    `override` wins verbatim when given; otherwise the UA names getdocs and its
+    version, with the project URL and — when supplied — the user's contact:
+        getdocs/0.1.0 (+https://github.com/jonbakerfish/getdocs; you@example.com)
+    """
+    if override:
+        return override
+    detail = PROJECT_URL if not contact else f"{PROJECT_URL}; {contact}"
+    return f"getdocs/{_version()} (+{detail})"